diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/allocator.rs | 48 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 127 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/callee.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 87 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/declare.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/errors.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 28 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/mono_item.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_.rs | 37 |
17 files changed, 355 insertions, 102 deletions
diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index fc9251dda82..668d9292705 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -4,6 +4,7 @@ use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_span::symbol::sym; use crate::debuginfo; use crate::llvm::{self, False, True}; @@ -14,6 +15,7 @@ pub(crate) unsafe fn codegen( module_llvm: &mut ModuleLlvm, module_name: &str, kind: AllocatorKind, + alloc_error_handler_kind: AllocatorKind, ) { let llcx = &*module_llvm.llcx; let llmod = module_llvm.llmod(); @@ -98,6 +100,52 @@ pub(crate) unsafe fn codegen( llvm::LLVMDisposeBuilder(llbuilder); } + // rust alloc error handler + let args = [usize, usize]; // size, align + + let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); + let name = "__rust_alloc_error_handler"; + let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + // -> ! DIFlagNoReturn + let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); + } + + let callee = alloc_error_handler_kind.fn_name(sym::oom); + let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + // -> ! DIFlagNoReturn + attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::<Vec<_>>(); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + llvm::LLVMSetTailCall(ret, True); + llvm::LLVMBuildRetVoid(llbuilder); + llvm::LLVMDisposeBuilder(llbuilder); + // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1a3865360a3..2a6ad1be763 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -236,8 +236,22 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::Nvptx64 => {} InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} + InlineAsmArch::LoongArch64 => { + constraints.extend_from_slice(&[ + "~{$fcc0}".to_string(), + "~{$fcc1}".to_string(), + "~{$fcc2}".to_string(), + "~{$fcc3}".to_string(), + "~{$fcc4}".to_string(), + "~{$fcc5}".to_string(), + "~{$fcc6}".to_string(), + "~{$fcc7}".to_string(), + ]); + } InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} + InlineAsmArch::S390x => { + constraints.push("~{cc}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} @@ -442,9 +456,9 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, None, v, inputs, None) + bx.call(fty, None, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors @@ -633,6 +647,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", @@ -719,6 +735,7 @@ fn modifier_to_llvm( } } InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, InlineAsmRegClass::PowerPC(_) => None, @@ -803,6 +820,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 12da21dc477..a6416e9540c 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -198,7 +198,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { "arm" => ("arm", "--32"), _ => panic!("unsupported arch {}", sess.target.arch), }; - let result = std::process::Command::new(dlltool) + let result = std::process::Command::new(&dlltool) .args([ "-d", def_file_path.to_str().unwrap(), @@ -218,9 +218,13 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match result { Err(e) => { - sess.emit_fatal(ErrorCallingDllTool { error: e }); + sess.emit_fatal(ErrorCallingDllTool { + dlltool_path: dlltool.to_string_lossy(), + error: e, + }); } - Ok(output) if !output.status.success() => { + // dlltool returns '0' on failure, so check for error output instead. + Ok(output) if !output.stderr.is_empty() => { sess.emit_fatal(DlltoolFailImportLibrary { stdout: String::from_utf8_lossy(&output.stdout), stderr: String::from_utf8_lossy(&output.stderr), @@ -431,7 +435,7 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.cg.dlltool { return dlltool_path.clone().into_os_string(); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 7136f750f39..ca2eab28f87 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -31,6 +31,7 @@ use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -363,6 +364,15 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void line: opt.line, column: opt.column, pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, message: &opt.message, }); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 6819a2af09d..4d0bcd53d15 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -15,14 +15,15 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; -use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use std::borrow::Cow; use std::ffi::CStr; use std::iter; @@ -216,6 +217,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -230,19 +232,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let invoke = unsafe { @@ -989,13 +985,20 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } + fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 1); + self.add_clause(landing_pad, self.const_array(self.type_i8p(), &[])); + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) + } + fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); let mut exn = self.const_poison(ty); @@ -1183,6 +1186,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1195,19 +1199,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if let Some(fn_abi) = fn_abi && self.tcx.sess.is_sanitizer_kcfi_enabled() && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let call = unsafe { @@ -1456,7 +1454,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, None, f, args, None) + self.call(ty, None, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1518,7 +1516,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) }; let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) + self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None) } pub(crate) fn landing_pad( @@ -1535,4 +1533,71 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) } } + + // Emits CFI pointer type membership tests. + fn cfi_type_test( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) { + return; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + self.abort(); + self.unreachable(); + + self.switch_to_block(bb_pass); + } + } + + // Emits KCFI operand bundles. + fn kcfi_operand_bundle( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) -> Option<llvm::OperandBundleDef<'ll>> { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + kcfi_bundle + } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 6ee2a05ffd7..30a0cf1d019 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -27,7 +27,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); assert!(!instance.substs.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f0d729d4779..83101a85435 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -228,18 +228,29 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, + llvm::LLVMModFlagBehavior::Override, canonical_jump_tables, 1, ); } + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + enable_split_lto_unit, + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { let kcfi = "kcfi\0".as_ptr().cast(); llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 2f910c37d61..3fff112a020 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -93,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( let callee = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - callee, + ty::EarlyBinder(callee), ); let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); cx.dbg_scope_fn(callee, callee_fn_abi, None) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 824cffa28ba..c3f0a0033b0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -322,7 +322,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); + let (containing_scope, is_method) = get_containing_scope(self, instance); let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -378,8 +378,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( + // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because + // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition. + // When we use this `decl` below, the subprogram definition gets created at the CU level + // with a DW_AT_specification pointing back to the type's declaration. + let decl = is_method.then(|| unsafe { + llvm::LLVMRustDIBuilderCreateMethod( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + flags, + spflags & !DISPFlags::SPFlagDefinition, + template_parameters, + ) + }); + + return unsafe { + llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -394,9 +415,9 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { spflags, maybe_definition_llfn, template_parameters, - None, - ); - } + decl, + ) + }; fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, @@ -493,57 +514,53 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { names } + /// Returns a scope, plus `true` if that's a type scope for "class" methods, + /// otherwise `false` for plain namespace scopes. fn get_containing_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, - ) -> &'ll DIScope { + ) -> (&'ll DIScope, bool) { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - cx.tcx.type_of(impl_def_id).skip_binder(), + cx.tcx.type_of(impl_def_id), ); // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } + if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - _ => None, } } else { // For trait method impls we still use the "parallel namespace" // strategy - None } - }); + } - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + (scope, false) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 6a575095f7e..cc2a5d158be 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -20,7 +20,7 @@ use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi}; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use smallvec::SmallVec; /// Declare a function. @@ -132,12 +132,31 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.apply_attrs_llfn(self, llfn); if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); } if self.tcx.sess.is_sanitizer_kcfi_enabled() { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); + // LLVM KCFI does not support multiple !kcfi_type attachments + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); self.set_kcfi_type_metadata(llfn, kcfi_typeid); } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bae88d94293..6a9173ab450 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -67,7 +67,8 @@ pub(crate) struct ErrorWritingDEFFile { #[derive(Diagnostic)] #[diag(codegen_llvm_error_calling_dlltool)] -pub(crate) struct ErrorCallingDllTool { +pub(crate) struct ErrorCallingDllTool<'a> { + pub dlltool_path: Cow<'a, str>, pub error: std::io::Error, } @@ -195,6 +196,7 @@ pub(crate) struct FromLlvmOptimizationDiag<'a> { pub line: std::ffi::c_uint, pub column: std::ffi::c_uint, pub pass_name: &'a str, + pub kind: &'a str, pub message: &'a str, } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 012e25884ca..00d1796f210 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -110,6 +110,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call( simple_ty, None, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -444,7 +445,7 @@ fn try_intrinsic<'ll>( ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, None, try_func, &[data], None); + bx.call(try_func_ty, None, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -543,7 +544,7 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -587,7 +588,7 @@ fn codegen_msvc_try<'ll>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -595,7 +596,7 @@ fn codegen_msvc_try<'ll>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -604,7 +605,7 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -647,7 +648,7 @@ fn codegen_gnu_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -665,13 +666,13 @@ fn codegen_gnu_try<'ll>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], None); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -711,7 +712,7 @@ fn codegen_emcc_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -750,13 +751,13 @@ fn codegen_emcc_try<'ll>( let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, catch_data], None); + bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -1205,6 +1206,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let c = bx.call( fn_ty, None, + None, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -1423,6 +1425,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None, @@ -1564,6 +1567,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None, @@ -2037,7 +2041,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, None, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 00b2dc1287a..8305a0a4c28 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -115,10 +115,11 @@ impl ExtraBackendMethods for LlvmCodegenBackend { tcx: TyCtxt<'tcx>, module_name: &str, kind: AllocatorKind, + alloc_error_handler_kind: AllocatorKind, ) -> ModuleLlvm { let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); unsafe { - allocator::codegen(tcx, &mut module_llvm, module_name, kind); + allocator::codegen(tcx, &mut module_llvm, module_name, kind, alloc_error_handler_kind); } module_llvm } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index c95148013eb..61365e6dc4b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1987,6 +1987,21 @@ extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateMethod<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + SPFlags: DISPFlags, + TParam: &'a DIArray, + ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2249,7 +2264,7 @@ extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine, cpu: *const c_char); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 46692fd5e8b..2fbdab9f8ce 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -329,7 +329,14 @@ pub(crate) fn print(req: PrintRequest, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetCPUs => { + // SAFETY generate a C compatible string from a byte slice to pass + // the target CPU name into LLVM, the lifetime of the reference is + // at least as long as the C function + let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) + .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); + unsafe { llvm::LLVMRustPrintTargetCPUs(tm, cpu_cstring.as_ptr()) }; + } PrintRequest::TargetFeatures => print_target_features(sess, tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index d0ae36349df..59bdc60830f 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -48,7 +48,7 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); let lldecl = self.declare_fn(symbol_name, fn_abi); diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index bef4647f207..d3fad5699c8 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -291,8 +291,24 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn add_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); let v = [self.const_usize(0), typeid_metadata]; unsafe { llvm::LLVMGlobalSetMetadata( @@ -307,13 +323,28 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn typeid_metadata(&self, typeid: String) -> &'ll Value { - unsafe { + fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> { + Some(unsafe { llvm::LLVMMDStringInContext( self.llcx, typeid.as_ptr() as *const c_char, typeid.len() as c_uint, ) + }) + } + + fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) } } |