diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
30 files changed, 825 insertions, 540 deletions
diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 668d9292705..ad0636894b7 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -1,10 +1,12 @@ use crate::attributes; use libc::c_uint; -use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_ast::expand::allocator::{ + alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy, + ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, +}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; -use rustc_span::symbol::sym; use crate::debuginfo; use crate::llvm::{self, False, True}; @@ -29,75 +31,78 @@ pub(crate) unsafe fn codegen( let i8p = llvm::LLVMPointerType(i8, 0); let void = llvm::LLVMVoidTypeInContext(llcx); - for method in ALLOCATOR_METHODS { - let mut args = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { - AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } - AllocatorTy::Ptr => args.push(i8p), - AllocatorTy::Usize => args.push(usize), - - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), } - } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + let ty = llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ); + let name = global_fn_name(method.name); + let llfn = + llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } - }; - let ty = llvm::LLVMFunctionType( - output.unwrap_or(void), - args.as_ptr(), - args.len() as c_uint, - False, - ); - let name = format!("__rust_{}", method.name); - let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); - - if tcx.sess.target.default_hidden_visibility { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); - } - if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let callee = kind.fn_name(method.name); - let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); - - let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); - llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); - let args = args - .iter() - .enumerate() - .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) - .collect::<Vec<_>>(); - let ret = llvm::LLVMRustBuildCall( - llbuilder, - ty, - callee, - args.as_ptr(), - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - llvm::LLVMSetTailCall(ret, True); - if output.is_some() { - llvm::LLVMBuildRet(llbuilder, ret); - } else { - llvm::LLVMBuildRetVoid(llbuilder); + let callee = default_fn_name(method.name); + let callee = + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::<Vec<_>>(); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + llvm::LLVMSetTailCall(ret, True); + if output.is_some() { + llvm::LLVMBuildRet(llbuilder, ret); + } else { + llvm::LLVMBuildRetVoid(llbuilder); + } + llvm::LLVMDisposeBuilder(llbuilder); } - llvm::LLVMDisposeBuilder(llbuilder); } // rust alloc error handler @@ -118,13 +123,13 @@ pub(crate) unsafe fn codegen( attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } - let callee = alloc_error_handler_kind.fn_name(sym::oom); + let callee = alloc_error_handler_name(alloc_error_handler_kind); let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); // -> ! DIFlagNoReturn attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); @@ -156,6 +161,14 @@ pub(crate) unsafe fn codegen( let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); + let name = NO_ALLOC_SHIM_IS_UNSTABLE; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let llval = llvm::LLVMConstInt(i8, 0, False); + llvm::LLVMSetInitializer(ll_g, llval); + if tcx.sess.opts.debuginfo != DebugInfo::None { let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index d9f8170a3cf..2a6ad1be763 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -236,14 +236,31 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { InlineAsmArch::Nvptx64 => {} InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} InlineAsmArch::Hexagon => {} + InlineAsmArch::LoongArch64 => { + constraints.extend_from_slice(&[ + "~{$fcc0}".to_string(), + "~{$fcc1}".to_string(), + "~{$fcc2}".to_string(), + "~{$fcc3}".to_string(), + "~{$fcc4}".to_string(), + "~{$fcc5}".to_string(), + "~{$fcc6}".to_string(), + "~{$fcc7}".to_string(), + ]); + } InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} - InlineAsmArch::S390x => {} + InlineAsmArch::S390x => { + constraints.push("~{cc}".to_string()); + } InlineAsmArch::SpirV => {} InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} InlineAsmArch::Bpf => {} InlineAsmArch::Msp430 => { constraints.push("~{sr}".to_string()); } + InlineAsmArch::M68k => { + constraints.push("~{ccr}".to_string()); + } } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -381,7 +398,7 @@ impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { } unsafe { - llvm::LLVMRustAppendModuleInlineAsm( + llvm::LLVMAppendModuleInlineAsm( self.llmod, template_str.as_ptr().cast(), template_str.len(), @@ -439,9 +456,9 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, v, inputs, dest, catch, funclet) + bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) } else { - bx.call(fty, None, v, inputs, None) + bx.call(fty, None, None, v, inputs, None) }; // Store mark in a metadata node so we can map LLVM errors @@ -630,6 +647,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", @@ -671,6 +690,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a", + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -713,6 +735,7 @@ fn modifier_to_llvm( } } InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, InlineAsmRegClass::PowerPC(_) => None, @@ -768,6 +791,7 @@ fn modifier_to_llvm( InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::M68k(_) => None, InlineAsmRegClass::Err => unreachable!(), } } @@ -796,6 +820,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' cx.type_vector(cx.type_i64(), 2) } InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), @@ -839,6 +865,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(), + InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 651d644ebb6..6d00464e0a0 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -88,6 +88,9 @@ pub fn sanitize_attrs<'ll>( attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); } + if enabled.contains(SanitizerSet::SAFESTACK) { + attrs.push(llvm::AttributeKind::SanitizeSafeStack.create_attr(cx.llcx)); + } attrs } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index dd3268d7780..a6416e9540c 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -110,7 +110,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a> { // FIXME use ArArchiveBuilder on most targets again once reading thin archives is // implemented - if true || sess.target.arch == "wasm32" || sess.target.arch == "wasm64" { + if true { Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) } else { Box::new(ArArchiveBuilder::new(sess, get_llvm_object_symbols)) @@ -189,7 +189,16 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { path.push(lib_name); path }; - let result = std::process::Command::new(dlltool) + // dlltool target architecture args from: + // https://github.com/llvm/llvm-project-release-prs/blob/llvmorg-15.0.6/llvm/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp#L69 + let (dlltool_target_arch, dlltool_target_bitness) = match sess.target.arch.as_ref() { + "x86_64" => ("i386:x86-64", "--64"), + "x86" => ("i386", "--32"), + "aarch64" => ("arm64", "--64"), + "arm" => ("arm", "--32"), + _ => panic!("unsupported arch {}", sess.target.arch), + }; + let result = std::process::Command::new(&dlltool) .args([ "-d", def_file_path.to_str().unwrap(), @@ -197,6 +206,10 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { lib_name, "-l", output_path.to_str().unwrap(), + "-m", + dlltool_target_arch, + "-f", + dlltool_target_bitness, "--no-leading-underscore", "--temp-prefix", temp_prefix.to_str().unwrap(), @@ -205,9 +218,13 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match result { Err(e) => { - sess.emit_fatal(ErrorCallingDllTool { error: e }); + sess.emit_fatal(ErrorCallingDllTool { + dlltool_path: dlltool.to_string_lossy(), + error: e, + }); } - Ok(output) if !output.status.success() => { + // dlltool returns '0' on failure, so check for error output instead. + Ok(output) if !output.stderr.is_empty() => { sess.emit_fatal(DlltoolFailImportLibrary { stdout: String::from_utf8_lossy(&output.stdout), stderr: String::from_utf8_lossy(&output.stderr), @@ -418,28 +435,26 @@ fn string_to_io_error(s: String) -> io::Error { fn find_binutils_dlltool(sess: &Session) -> OsString { assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); - if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + if let Some(dlltool_path) = &sess.opts.cg.dlltool { return dlltool_path.clone().into_os_string(); } - let mut tool_name: OsString = if sess.host.arch != sess.target.arch { - // We are cross-compiling, so we need the tool with the prefix matching our target - if sess.target.arch == "x86" { - "i686-w64-mingw32-dlltool" - } else { - "x86_64-w64-mingw32-dlltool" - } + let tool_name: OsString = if sess.host.options.is_like_windows { + // If we're compiling on Windows, always use "dlltool.exe". + "dlltool.exe" } else { - // We are not cross-compiling, so we just want `dlltool` - "dlltool" + // On other platforms, use the architecture-specific name. + match sess.target.arch.as_ref() { + "x86_64" => "x86_64-w64-mingw32-dlltool", + "x86" => "i686-w64-mingw32-dlltool", + "aarch64" => "aarch64-w64-mingw32-dlltool", + + // For non-standard architectures (e.g., aarch32) fallback to "dlltool". + _ => "dlltool", + } } .into(); - if sess.host.options.is_like_windows { - // If we're compiling on Windows, add the .exe suffix - tool_name.push(".exe"); - } - // NOTE: it's not clear how useful it is to explicitly search PATH. for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { let full_path = dir.join(&tool_name); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index d2e01708a37..8b05af7bed9 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -25,7 +25,6 @@ use std::fs::File; use std::io; use std::iter; use std::path::Path; -use std::ptr; use std::slice; use std::sync::Arc; @@ -596,7 +595,7 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddModuleFlag( module.module_llvm.llmod(), llvm::LLVMModFlagBehavior::Error, - "LTOPostLink\0".as_ptr().cast(), + c"LTOPostLink".as_ptr().cast(), 1, ); } @@ -709,17 +708,6 @@ pub unsafe fn optimize_thin_module( let llmod = module.module_llvm.llmod(); save_temp_bitcode(cgcx, &module, "thin-lto-input"); - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - return Err(write::llvm_err(&diag_handler, LlvmError::MultipleSourceDiCompileUnit)); - } - // Up next comes the per-module local analyses that we do for Thin LTO. // Each of these functions is basically copied from the LLVM // implementation and then tailored to suit this implementation. Ideally @@ -766,43 +754,6 @@ pub unsafe fn optimize_thin_module( save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); - } - // Alright now that we've done everything related to the ThinLTO // analysis it's time to run some optimizations! Here we use the same // `run_pass_manager` as the "fat" LTO above except that we tell it to diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index a4ae1b01e86..53b4296802e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -31,6 +31,7 @@ use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -214,6 +215,8 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); + let force_emulated_tls = sess.target.force_emulated_tls; + Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; @@ -239,6 +242,7 @@ pub fn target_machine_factory( relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), + force_emulated_tls, ) }; @@ -360,6 +364,15 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void line: opt.line, column: opt.column, pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, message: &opt.message, }); } @@ -878,11 +891,11 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.module\0".as_ptr().cast(), + c"rustc.embedded.module".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + let section = if is_apple { c"__LLVM,__bitcode" } else { c".llvmbc" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); @@ -891,19 +904,19 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.cmdline\0".as_ptr().cast(), + c"rustc.embedded.cmdline".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + let section = if is_apple { c"__LLVM,__cmdline" } else { c".llvmcmd" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5b2bbdb4bde..2f7eb08ad3d 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -19,8 +19,6 @@ use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; -use cstr::cstr; - use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; @@ -110,11 +108,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // Create the llvm.used and llvm.compiler.used variables. if !cx.used_statics.borrow().is_empty() { - cx.create_used_variable_impl(cstr!("llvm.used"), &*cx.used_statics.borrow()); + cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow()); } if !cx.compiler_used_statics.borrow().is_empty() { cx.create_used_variable_impl( - cstr!("llvm.compiler.used"), + c"llvm.compiler.used", &*cx.compiler_used_statics.borrow(), ); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0f33b985489..b4aa001547c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2,11 +2,10 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock}; +use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; use libc::{c_char, c_uint}; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -15,16 +14,16 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; -use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi; +use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use std::borrow::Cow; -use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; @@ -44,13 +43,10 @@ impl Drop for Builder<'_, '_, '_> { } } -// FIXME(eddyb) use a checked constructor when they become `const fn`. -const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; - /// Empty string, to be used where LLVM expects an instruction name, indicating /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); +const UNNAMED: *const c_char = c"".as_ptr(); impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value; @@ -216,6 +212,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn invoke( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -230,19 +227,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let invoke = unsafe { @@ -490,7 +481,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); if place.layout.is_zst() { - return OperandRef::new_zst(self, place.layout); + return OperandRef::zero_sized(place.layout); } #[instrument(level = "trace", skip(bx))] @@ -841,7 +832,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + unsafe { + llvm::LLVMBuildIntCast2( + self.llbuilder, + val, + dest_ty, + if is_signed { True } else { False }, + UNNAMED, + ) + } } fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { @@ -981,16 +980,23 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); - let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { llvm::LLVMSetCleanup(landing_pad, llvm::True); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } + fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); + let landing_pad = self.landing_pad(ty, pers_fn, 1); + self.add_clause(landing_pad, self.const_array(self.type_i8p(), &[])); + (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) + } + fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); - let mut exn = self.const_undef(ty); + let mut exn = self.const_poison(ty); exn = self.insert_value(exn, exn0, 0); exn = self.insert_value(exn, exn1, 1); unsafe { @@ -999,14 +1005,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("cleanuppad"); let ret = unsafe { - llvm::LLVMRustBuildCleanupPad( + llvm::LLVMBuildCleanupPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), - name.as_ptr(), + args.len() as c_uint, + c"cleanuppad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) @@ -1014,20 +1019,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { unsafe { - llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) .expect("LLVM does not have support for cleanupret"); } } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("catchpad"); let ret = unsafe { - llvm::LLVMRustBuildCatchPad( + llvm::LLVMBuildCatchPad( self.llbuilder, parent, - args.len() as c_uint, args.as_ptr(), - name.as_ptr(), + args.len() as c_uint, + c"catchpad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for catchpad")) @@ -1039,20 +1043,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unwind: Option<&'ll BasicBlock>, handlers: &[&'ll BasicBlock], ) -> &'ll Value { - let name = cstr!("catchswitch"); let ret = unsafe { - llvm::LLVMRustBuildCatchSwitch( + llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, handlers.len() as c_uint, - name.as_ptr(), + c"catchswitch".as_ptr(), ) }; let ret = ret.expect("LLVM does not have support for catchswitch"); for handler in handlers { unsafe { - llvm::LLVMRustAddHandler(ret, handler); + llvm::LLVMAddHandler(ret, handler); } } ret @@ -1175,6 +1178,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1187,19 +1191,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); let mut bundles = vec![funclet_bundle]; - // Set KCFI operand bundle - let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; - let kcfi_bundle = - if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap()); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; - if kcfi_bundle.is_some() { - let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); - } + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + bundles.push(kcfi_bundle); bundles.retain(|bundle| bundle.is_some()); let call = unsafe { @@ -1376,8 +1374,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { - let ret = - unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; ret.expect("LLVM does not have support for catchret") } @@ -1449,7 +1446,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(intrinsic); - self.call(ty, None, f, args, None) + self.call(ty, None, None, f, args, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { @@ -1511,7 +1508,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) }; let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); - self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) + self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None) } pub(crate) fn landing_pad( @@ -1528,4 +1525,71 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) } } + + // Emits CFI pointer type membership tests. + fn cfi_type_test( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) { + return; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + + // Test whether the function pointer is associated with the type identifier. + let cond = self.type_test(llfn, typeid_metadata); + let bb_pass = self.append_sibling_block("type_test.pass"); + let bb_fail = self.append_sibling_block("type_test.fail"); + self.cond_br(cond, bb_pass, bb_fail); + + self.switch_to_block(bb_fail); + self.abort(); + self.unreachable(); + + self.switch_to_block(bb_pass); + } + } + + // Emits KCFI operand bundles. + fn kcfi_operand_bundle( + &mut self, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + ) -> Option<llvm::OperandBundleDef<'ll>> { + let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; + let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() { + if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) { + return None; + } + + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; + kcfi_bundle + } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 6ee2a05ffd7..4b9ca2e7d19 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -27,7 +27,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); assert!(!instance.substs.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { @@ -94,11 +94,11 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // LLVM will prefix the name with `__imp_`. Ideally, we'd like the // existing logic below to set the Storage Class, but it has an // exemption for MinGW for backwards compatability. - let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi); + let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } llfn } else { - cx.declare_fn(sym, fn_abi) + cx.declare_fn(sym, fn_abi, Some(instance)) }; debug!("get_fn: not casting pointer!"); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b0a9a30ab46..a3910fef954 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -8,16 +8,15 @@ use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_ast::Mutability; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use rustc_target::spec::Target; use libc::{c_char, c_uint}; @@ -130,6 +129,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMGetUndef(t) } } + fn const_poison(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetPoison(t) } + } + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { unsafe { llvm::LLVMConstInt(t, i as u64, True) } } @@ -257,7 +260,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { let hash = self.tcx.with_stable_hashing_context(|mut hcx| { let mut hasher = StableHasher::new(); alloc.hash_stable(&mut hcx, &mut hasher); - hasher.finish::<u128>() + hasher.finish::<Hash128>() }); llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes()); } @@ -303,38 +306,24 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { const_alloc_to_llvm(self, alloc) } - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, &'ll Value> { - let alloc_align = alloc.inner().align; - assert_eq!(alloc_align, layout.align.abi); - let llty = self.type_ptr_to(layout.llvm_type(self)); - let llval = if layout.size == Size::ZERO { - let llval = self.const_usize(alloc_align.bytes()); - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - let init = const_alloc_to_llvm(self, alloc); - let base_addr = self.static_addr_of(init, alloc_align, None); - - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p()), - &self.const_usize(offset.bytes()), - 1, - ) - }; - self.const_bitcast(llval, llty) - }; - PlaceRef::new_sized(llval, layout) - } - fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { consts::ptrcast(val, ty) } + + fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + self.const_bitcast(val, ty) + } + + fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { + unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p()), + &self.const_usize(offset.bytes()), + 1, + ) + } + } } /// Get the [LLVM type][Type] of a [`Value`]. @@ -374,8 +363,7 @@ pub(crate) fn get_dllimport<'tcx>( name: &str, ) -> Option<&'tcx DllImport> { tcx.native_library(id) - .map(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) - .flatten() + .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } pub(crate) fn is_mingw_gnu_toolchain(target: &Target) -> bool { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9116e71beac..2087754c66b 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,13 +1,13 @@ use crate::base; use crate::common::{self, CodegenCx}; use crate::debuginfo; -use crate::errors::{InvalidMinimumAlignment, SymbolAlreadyDefined}; +use crate::errors::{ + InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, +}; use crate::llvm::{self, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; -use libc::c_uint; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -20,7 +20,9 @@ use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, span_bug}; use rustc_session::config::Lto; -use rustc_target::abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; +use rustc_target::abi::{ + Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange, +}; use std::ops::Range; pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { @@ -130,9 +132,14 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: if let Some(min) = cx.sess().target.min_global_align { match Align::from_bits(min) { Ok(min) => align = align.max(min), - Err(err) => { - cx.sess().emit_err(InvalidMinimumAlignment { err }); - } + Err(err) => match err { + AlignFromBytesError::NotPowerOfTwo(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); + } + AlignFromBytesError::TooLarge(align) => { + cx.sess().emit_err(InvalidMinimumAlignmentTooLarge { align }); + } + }, } } unsafe { @@ -474,9 +481,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { .all(|&byte| byte == 0); let sect_name = if all_bytes_are_zero { - cstr!("__DATA,__thread_bss") + c"__DATA,__thread_bss" } else { - cstr!("__DATA,__thread_data") + c"__DATA,__thread_data" }; llvm::LLVMSetSection(g, sect_name.as_ptr()); } @@ -486,10 +493,10 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // go into custom sections of the wasm executable. if self.tcx.sess.target.is_like_wasm { if let Some(section) = attrs.link_section { - let section = llvm::LLVMMDStringInContext( + let section = llvm::LLVMMDStringInContext2( self.llcx, section.as_str().as_ptr().cast(), - section.as_str().len() as c_uint, + section.as_str().len(), ); assert!(alloc.provenance().ptrs().is_empty()); @@ -498,17 +505,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { // as part of the interpreter execution). let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); - let alloc = llvm::LLVMMDStringInContext( - self.llcx, - bytes.as_ptr().cast(), - bytes.len() as c_uint, - ); + let alloc = + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); let data = [section, alloc]; - let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); + let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, - "wasm.custom_sections\0".as_ptr().cast(), - meta, + c"wasm.custom_sections".as_ptr().cast(), + val, ); } } else { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f0d729d4779..e8a7afcc632 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use crate::llvm_util; use crate::type_::Type; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; @@ -224,25 +223,42 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + c"RtLibUseGOT".as_ptr().cast(), + 1, + ); } - if sess.is_sanitizer_cfi_enabled() { - // FIXME(rcvalle): Add support for non canonical jump tables. - let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); - // FIXME(rcvalle): Add it with Override behavior flag. + // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) + if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { llvm::LLVMRustAddModuleFlag( llmod, - llvm::LLVMModFlagBehavior::Warning, - canonical_jump_tables, + llvm::LLVMModFlagBehavior::Override, + c"CFI Canonical Jump Tables".as_ptr().cast(), 1, ); } + // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) + if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"EnableSplitLTOUnit".as_ptr().cast(), + 1, + ); + } + + // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = "kcfi\0".as_ptr().cast(); - llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + c"kcfi".as_ptr().cast(), + 1, + ); } // Control Flow Guard is currently only supported by the MSVC linker on Windows. @@ -254,7 +270,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 1, ) } @@ -263,7 +279,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 2, ) } @@ -281,26 +297,26 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, behavior, - "branch-target-enforcement\0".as_ptr().cast(), + c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address\0".as_ptr().cast(), + c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address-all\0".as_ptr().cast(), + c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address-with-bkey\0".as_ptr().cast(), + c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); } else { @@ -316,7 +332,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-branch\0".as_ptr().cast(), + c"cf-protection-branch".as_ptr().cast(), 1, ) } @@ -324,7 +340,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-return\0".as_ptr().cast(), + c"cf-protection-return".as_ptr().cast(), 1, ) } @@ -333,7 +349,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Error, - "Virtual Function Elim\0".as_ptr().cast(), + c"Virtual Function Elim".as_ptr().cast(), 1, ); } @@ -465,14 +481,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { - let section = cstr!("llvm.metadata"); let array = self.const_array(self.type_ptr_to(self.type_i8()), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, section.as_ptr()); + llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); } } } @@ -958,9 +973,9 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let LayoutError::SizeOverflow(_) = err { - self.sess().emit_fatal(Spanned { span, node: err }) + self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + span_bug!(span, "failed to get layout for `{ty}`: {err:?}") } } } @@ -980,21 +995,12 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { - span_bug!( - span, - "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", - sig, - extra_args, - err - ); + span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",); } FnAbiRequest::OfInstance { instance, extra_args } => { span_bug!( span, - "`fn_abi_of_instance({}, {:?})` failed: {}", - instance, - extra_args, - err + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", ); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 240a9d2f371..21a1ac34844 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -163,7 +163,7 @@ impl CoverageMapGenerator { counter_regions.sort_unstable_by_key(|(_counter, region)| *region); for (counter, region) in counter_regions { let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; - let same_file = current_file_name.map_or(false, |p| p == file_name); + let same_file = current_file_name.is_some_and(|p| p == file_name); if !same_file { if current_file_name.is_some() { current_file_id += 1; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 3dc0ac03312..cd261293e9b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -207,6 +207,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< )), ty::List::empty(), ), + None, ); llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 99e4ded62f1..64961baf272 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. // FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. @@ -93,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( let callee = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - callee, + ty::EarlyBinder::bind(callee), ); let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); cx.dbg_scope_fn(callee, callee_fn_abi, None) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 80fd9726fc7..8be54b7eb71 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -5,14 +5,13 @@ use crate::llvm; use crate::builder::Builder; use crate::common::CodegenCx; use crate::value::Value; +use rustc_ast::attr; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_middle::bug; +use rustc_middle::{bug, middle::debugger_visualizer::DebuggerVisualizerType}; use rustc_session::config::{CrateType, DebugInfo}; - use rustc_span::symbol::sym; -use rustc_span::DebuggerVisualizerType; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -22,9 +21,9 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); // Load just the first byte as that's all that's necessary to force // LLVM to keep around the reference to the global. - let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + let volatile_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); unsafe { - llvm::LLVMSetAlignment(volative_load_instruction, 1); + llvm::LLVMSetAlignment(volatile_load_instruction, 1); } } } @@ -39,7 +38,6 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; section_var.unwrap_or_else(|| { - let section_name = b".debug_gdb_scripts\0"; let mut section_contents = Vec::new(); // Add the pretty printers for the standard library first. @@ -72,7 +70,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); + llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast()); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); @@ -87,7 +85,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { let omit_gdb_pretty_printer_section = - cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index c1b3f34e5a6..166454d3ae7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -20,7 +20,6 @@ use crate::llvm::debuginfo::{ }; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; @@ -29,7 +28,6 @@ use rustc_hir::def::CtorKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::{ self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; @@ -176,15 +174,14 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (thin_pointer_size, thin_pointer_align) = - cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let data_layout = &cx.tcx.data_layout; let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); match fat_pointer_kind(cx, pointee_type) { None => { // This is a thin pointer. Create a regular pointer type and give it the correct name. debug_assert_eq!( - (thin_pointer_size, thin_pointer_align), + (data_layout.pointer_size, data_layout.pointer_align.abi), cx.size_and_align_of(ptr_type), "ptr_type={}, pointee_type={}", ptr_type, @@ -195,8 +192,8 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), pointee_type_di_node, - thin_pointer_size.bits(), - thin_pointer_align.bits() as u32, + data_layout.pointer_size.bits(), + data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. ptr_type_debuginfo_name.as_ptr().cast(), ptr_type_debuginfo_name.len(), @@ -808,14 +805,12 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( name_in_debuginfo.push(codegen_unit_name); debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); - let rustc_producer = - format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + let rustc_producer = format!("rustc version {}", tcx.sess.cfg_version); // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. let producer = format!("clang LLVM ({})", rustc_producer); let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); - let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames @@ -831,24 +826,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( } .unwrap_or_default(); let split_name = split_name.to_str().unwrap(); - - // FIXME(#60020): - // - // This should actually be - // - // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); - // - // That is, we should set LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we keep - // the emission kind as `FullDebug`. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - let kind = DebugEmissionKind::FullDebug; - assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( @@ -869,7 +847,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( producer.as_ptr().cast(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, - flags.as_ptr().cast(), + c"".as_ptr().cast(), 0, // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final @@ -882,8 +860,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); if tcx.sess.opts.unstable_opts.profile { - let cu_desc_metadata = - llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); let default_gcda_path = &output_filenames.with_extension("gcda"); let gcda_path = tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); @@ -891,20 +867,16 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let gcov_cu_info = [ path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), path_to_mdstring(debug_context.llcontext, gcda_path), - cu_desc_metadata, + unit_metadata, ]; - let gcov_metadata = llvm::LLVMMDNodeInContext( + let gcov_metadata = llvm::LLVMMDNodeInContext2( debug_context.llcontext, gcov_cu_info.as_ptr(), - gcov_cu_info.len() as c_uint, + gcov_cu_info.len(), ); + let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - llvm_gcov_ident.as_ptr(), - gcov_metadata, - ); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); } // Insert `llvm.ident` metadata on the wasm targets since that will @@ -917,7 +889,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); llvm::LLVMAddNamedMetadataOperand( debug_context.llmod, - cstr!("llvm.ident").as_ptr(), + c"llvm.ident".as_ptr(), llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), ); } @@ -925,15 +897,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( return unit_metadata; }; - fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll llvm::Metadata { let path_str = path_to_c_string(path); - unsafe { - llvm::LLVMMDStringInContext( - llcx, - path_str.as_ptr(), - path_str.as_bytes().len() as c_uint, - ) - } + unsafe { llvm::LLVMMDStringInContext2(llcx, path_str.as_ptr(), path_str.as_bytes().len()) } } } @@ -1211,12 +1177,12 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( let names = get_parameter_names(cx, generics); let template_params: SmallVec<_> = iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_di_node = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -1224,10 +1190,8 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( name.len(), actual_type_di_node, ) - }) - } else { - None - } + } + }) }) .collect(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 69443b9b828..ecb0912d328 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::{ traits::ConstMethods, }; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::{ bug, ty::{ @@ -62,7 +62,7 @@ const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0; /// In CPP-like mode, we generate a union with a field for each variant and an /// explicit tag field. The field of each variant has a struct type -/// that encodes the discrimiant of the variant and it's data layout. +/// that encodes the discriminant of the variant and it's data layout. /// The union also has a nested enumeration type that is only used for encoding /// variant names in an efficient way. Its enumerator values do _not_ correspond /// to the enum's discriminant values. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 54e850f2599..9e0e847a155 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -3,10 +3,10 @@ use rustc_codegen_ssa::debuginfo::{ wants_c_like_enum_debuginfo, }; use rustc_hir::def::CtorKind; -use rustc_index::vec::IndexVec; +use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + mir::{GeneratorLayout, GeneratorSavedLocal}, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, @@ -14,7 +14,9 @@ use rustc_middle::{ }, }; use rustc_span::Symbol; -use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use rustc_target::abi::{ + FieldIdx, HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants, +}; use std::borrow::Cow; use crate::{ @@ -272,7 +274,8 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( .map(|field_index| { let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) { // Fields have names - Cow::from(variant_def.fields[field_index].name.as_str()) + let field = &variant_def.fields[FieldIdx::from_usize(field_index)]; + Cow::from(field.name.as_str()) } else { // Tuple-like super::tuple_field_name(field_index) @@ -320,7 +323,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexVec<GeneratorSavedLocal, Option<Symbol>>, + state_specific_upvar_names: &IndexSlice<GeneratorSavedLocal, Option<Symbol>>, common_upvar_names: &[String], ) -> &'ll DIType { let variant_name = GeneratorSubsts::variant_name(variant_index); @@ -353,7 +356,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) .map(|field_index| { let generator_saved_local = generator_layout.variant_fields[variant_index] - [Field::from_usize(field_index)]; + [FieldIdx::from_usize(field_index)]; let field_name_maybe = state_specific_upvar_names[generator_saved_local]; let field_name = field_name_maybe .as_ref() diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 5392534cfcb..aa7ae9355bc 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -21,12 +21,13 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::Hash128; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; @@ -61,7 +62,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<Option<(u128, SourceFileHash)>, &'ll DIFile>>, + created_files: RefCell<FxHashMap<Option<(Hash128, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -112,7 +113,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "Dwarf Version\0".as_ptr().cast(), + c"Dwarf Version".as_ptr().cast(), dwarf_version, ); } else { @@ -120,17 +121,16 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "CodeView\0".as_ptr().cast(), + c"CodeView".as_ptr().cast(), 1, ) } // Prevent bitcode readers from deleting the debug info. - let ptr = "Debug Info Version\0".as_ptr(); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - ptr.cast(), + c"Debug Info Version".as_ptr().cast(), llvm::LLVMRustDebugMetadataVersion(), ); } @@ -209,8 +209,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { - let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc); } } @@ -322,7 +321,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let tcx = self.tcx; let def_id = instance.def_id(); - let containing_scope = get_containing_scope(self, instance); + let (containing_scope, is_method) = get_containing_scope(self, instance); let span = tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); @@ -378,8 +377,29 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - unsafe { - return llvm::LLVMRustDIBuilderCreateFunction( + // When we're adding a method to a type DIE, we only want a DW_AT_declaration there, because + // LLVM LTO can't unify type definitions when a child DIE is a full subprogram definition. + // When we use this `decl` below, the subprogram definition gets created at the CU level + // with a DW_AT_specification pointing back to the type's declaration. + let decl = is_method.then(|| unsafe { + llvm::LLVMRustDIBuilderCreateMethod( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + flags, + spflags & !DISPFlags::SPFlagDefinition, + template_parameters, + ) + }); + + return unsafe { + llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -394,15 +414,15 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { spflags, maybe_definition_llfn, template_parameters, - None, - ); - } + decl, + ) + }; fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, ) -> &'ll DIArray { - if cx.sess().opts.debuginfo == DebugInfo::Limited { + if cx.sess().opts.debuginfo != DebugInfo::Full { return create_DIArray(DIB(cx), &[]); } @@ -461,12 +481,12 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { let names = get_parameter_names(cx, generics); iter::zip(substs, names) .filter_map(|(kind, name)| { - if let GenericArgKind::Type(ty) = kind.unpack() { + kind.as_type().map(|ty| { let actual_type = cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); let actual_type_metadata = type_di_node(cx, actual_type); let name = name.as_str(); - Some(unsafe { + unsafe { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, @@ -474,10 +494,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { name.len(), actual_type_metadata, )) - }) - } else { - None - } + } + }) }) .collect() } else { @@ -495,57 +513,53 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { names } + /// Returns a scope, plus `true` if that's a type scope for "class" methods, + /// otherwise `false` for plain namespace scopes. fn get_containing_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, - ) -> &'ll DIScope { + ) -> (&'ll DIScope, bool) { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - cx.tcx.type_of(impl_def_id).skip_binder(), + cx.tcx.type_of(impl_def_id), ); // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - match impl_self_ty.kind() { - ty::Adt(def, ..) if !def.is_box() => { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full - && !impl_self_ty.needs_subst() - { - Some(type_di_node(cx, impl_self_ty)) - } else { - Some(namespace::item_namespace(cx, def.did())) - } + if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - _ => None, } } else { // For trait method impls we still use the "parallel namespace" // strategy - None } - }); + } - self_type.unwrap_or_else(|| { - namespace::item_namespace( - cx, - DefId { - krate: instance.def_id().krate, - index: cx - .tcx - .def_key(instance.def_id()) - .parent - .expect("get_containing_scope: missing parent?"), - }, - ) - }) + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + (scope, false) } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 6a575095f7e..164b12cf8d4 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -19,8 +19,11 @@ use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; -use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi}; +use rustc_middle::ty::{Instance, Ty}; +use rustc_symbol_mangling::typeid::{ + kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance, + TypeIdOptions, +}; use smallvec::SmallVec; /// Declare a function. @@ -116,7 +119,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + pub fn declare_fn( + &self, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: Option<Instance<'tcx>>, + ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); // Function addresses in Rust are never significant, allowing functions to @@ -132,13 +140,54 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.apply_attrs_llfn(self, llfn); if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi); - self.set_type_metadata(llfn, typeid); + if let Some(instance) = instance { + let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_instance( + self.tcx, + &instance, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } else { + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } } if self.tcx.sess.is_sanitizer_kcfi_enabled() { - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); - self.set_kcfi_type_metadata(llfn, kcfi_typeid); + // LLVM KCFI does not support multiple !kcfi_type attachments + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } + + if let Some(instance) = instance { + let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } else { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } } llfn diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index bae88d94293..44869ced1ae 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -50,9 +50,15 @@ pub(crate) struct SymbolAlreadyDefined<'a> { } #[derive(Diagnostic)] -#[diag(codegen_llvm_invalid_minimum_alignment)] -pub(crate) struct InvalidMinimumAlignment { - pub err: String, +#[diag(codegen_llvm_invalid_minimum_alignment_not_power_of_two)] +pub(crate) struct InvalidMinimumAlignmentNotPowerOfTwo { + pub align: u64, +} + +#[derive(Diagnostic)] +#[diag(codegen_llvm_invalid_minimum_alignment_too_large)] +pub(crate) struct InvalidMinimumAlignmentTooLarge { + pub align: u64, } #[derive(Diagnostic)] @@ -67,7 +73,8 @@ pub(crate) struct ErrorWritingDEFFile { #[derive(Diagnostic)] #[diag(codegen_llvm_error_calling_dlltool)] -pub(crate) struct ErrorCallingDllTool { +pub(crate) struct ErrorCallingDllTool<'a> { + pub dlltool_path: Cow<'a, str>, pub error: std::io::Error, } @@ -195,6 +202,7 @@ pub(crate) struct FromLlvmOptimizationDiag<'a> { pub line: std::ffi::c_uint, pub column: std::ffi::c_uint, pub pass_name: &'a str, + pub kind: &'a str, pub message: &'a str, } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9c921989ca9..4e28034a850 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -110,6 +110,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call( simple_ty, None, + None, simple_fn, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -378,7 +379,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } - _ => bug!("unknown intrinsic '{}'", name), + _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), }; if !fn_abi.ret.is_ignore() { @@ -444,7 +445,7 @@ fn try_intrinsic<'ll>( ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.call(try_func_ty, None, try_func, &[data], None); + bx.call(try_func_ty, None, None, try_func, &[data], None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. let ret_align = bx.tcx().data_layout.i32_align.abi; @@ -543,7 +544,7 @@ fn codegen_msvc_try<'ll>( let ptr_align = bx.tcx().data_layout.pointer_align.abi; let slot = bx.alloca(bx.type_i8p(), ptr_align); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); bx.switch_to_block(normal); bx.ret(bx.const_i32(0)); @@ -587,7 +588,7 @@ fn codegen_msvc_try<'ll>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_i8p(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -595,7 +596,7 @@ fn codegen_msvc_try<'ll>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_i8p()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet)); + bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -604,7 +605,7 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -647,7 +648,7 @@ fn codegen_gnu_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -665,13 +666,13 @@ fn codegen_gnu_try<'ll>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, ptr], None); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -711,7 +712,7 @@ fn codegen_emcc_try<'ll>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); - bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None); + bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); bx.switch_to_block(then); bx.ret(bx.const_i32(0)); @@ -750,13 +751,13 @@ fn codegen_emcc_try<'ll>( let catch_data = bx.bitcast(catch_data, bx.type_i8p()); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, catch_func, &[data, catch_data], None); + bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); bx.ret(bx.const_i32(1)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None); + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); let i32_align = bx.tcx().data_layout.i32_align.abi; bx.store(ret, dest, i32_align); } @@ -771,7 +772,7 @@ fn gen_fn<'ll, 'tcx>( ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); + let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. @@ -1205,6 +1206,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let c = bx.call( fn_ty, None, + None, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None, @@ -1423,6 +1425,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None, @@ -1564,6 +1567,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let v = bx.call( fn_ty, None, + None, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None, @@ -2037,7 +2041,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - let v = bx.call(fn_ty, None, f, &[lhs, rhs], None); + let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); return Ok(v); } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 8dafe1b750b..ff9909c720e 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -10,7 +10,8 @@ #![feature(iter_intersperse)] #![feature(let_chains)] #![feature(never_type)] -#![feature(once_cell)] +#![feature(impl_trait_in_assoc_type)] +#![feature(c_str_literals)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] @@ -35,10 +36,10 @@ use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; -use rustc_macros::fluent_messages; +use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; use rustc_session::Session; @@ -70,7 +71,7 @@ mod declare; mod errors; mod intrinsic; -// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +// The following is a workaround that replaces `pub mod llvm;` and that fixes issue 53912. #[path = "llvm/mod.rs"] mod llvm_; pub mod llvm { @@ -361,12 +362,12 @@ impl CodegenBackend for LlvmCodegenBackend { .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") .join(sess); - sess.time("llvm_dump_timing_file", || { - if sess.opts.unstable_opts.llvm_time_trace { + if sess.opts.unstable_opts.llvm_time_trace { + sess.time("llvm_dump_timing_file", || { let file_name = outputs.with_extension("llvm_timings.json"); llvm_util::time_trace_profiler_finish(&file_name); - } - }); + }); + } Ok((codegen_results, work_products)) } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 509cb0fef56..6ef3418cc5f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -196,6 +196,7 @@ pub enum AttributeKind { AllocSize = 37, AllocatedPointer = 38, AllocAlign = 39, + SanitizeSafeStack = 40, } /// LLVMIntPredicate @@ -552,6 +553,7 @@ pub enum ArchiveKind { K_BSD, K_DARWIN, K_COFF, + K_AIXBIG, } // LLVMRustThinLTOData @@ -679,7 +681,9 @@ pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_v pub mod coverageinfo { use super::coverage_map; - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegionKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum RegionKind { @@ -713,7 +717,9 @@ pub mod coverageinfo { /// array", encoded separately), and source location (start and end positions of the represented /// code region). /// - /// Matches LLVMRustCounterMappingRegion. + /// Corresponds to struct `llvm::coverage::CounterMappingRegion`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegion`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct CounterMappingRegion { @@ -946,15 +952,27 @@ pub mod debuginfo { NoDebug, FullDebug, LineTablesOnly, + DebugDirectivesOnly, } impl DebugEmissionKind { pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. use rustc_session::config::DebugInfo; match kind { DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, - DebugInfo::Full => DebugEmissionKind::FullDebug, + DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly, + DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, } } } @@ -1006,7 +1024,7 @@ extern "C" { pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); /// See Module::setModuleInlineAsm. - pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t); /// See llvm::LLVMTypeKind::getTypeID. pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; @@ -1053,7 +1071,7 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values pub fn LLVMTypeOf(Val: &Value) -> &Type; @@ -1069,9 +1087,15 @@ extern "C" { // Operations on constants of any type pub fn LLVMConstNull(Ty: &Type) -> &Value; pub fn LLVMGetUndef(Ty: &Type) -> &Value; + pub fn LLVMGetPoison(Ty: &Type) -> &Value; // Operations on metadata + // FIXME: deprecated, replace with LLVMMDStringInContext2 pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + + pub fn LLVMMDStringInContext2(C: &Context, Str: *const c_char, SLen: size_t) -> &Metadata; + + // FIXME: deprecated, replace with LLVMMDNodeInContext2 pub fn LLVMMDNodeInContext<'a>( C: &'a Context, Vals: *const &'a Value, @@ -1110,6 +1134,8 @@ extern "C" { Packed: Bool, ) -> &'a Value; + // FIXME: replace with LLVMConstArray2 when bumped minimal version to llvm-17 + // https://github.com/llvm/llvm-project/commit/35276f16e5a2cae0dfb49c0fbf874d4d2f177acc pub fn LLVMConstArray<'a>( ElementTy: &'a Type, ConstantVals: *const &'a Value, @@ -1249,7 +1275,7 @@ extern "C" { pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); // Metadata - pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: &'a Metadata); // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1289,38 +1315,38 @@ extern "C" { pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; - pub fn LLVMRustBuildCleanupPad<'a>( + pub fn LLVMBuildCleanupPad<'a>( B: &Builder<'a>, ParentPad: Option<&'a Value>, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCleanupRet<'a>( + pub fn LLVMBuildCleanupRet<'a>( B: &Builder<'a>, CleanupPad: &'a Value, - UnwindBB: Option<&'a BasicBlock>, + BB: Option<&'a BasicBlock>, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchPad<'a>( + pub fn LLVMBuildCatchPad<'a>( B: &Builder<'a>, ParentPad: &'a Value, - ArgCnt: c_uint, Args: *const &'a Value, + NumArgs: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchRet<'a>( + pub fn LLVMBuildCatchRet<'a>( B: &Builder<'a>, - Pad: &'a Value, + CatchPad: &'a Value, BB: &'a BasicBlock, ) -> Option<&'a Value>; - pub fn LLVMRustBuildCatchSwitch<'a>( + pub fn LLVMBuildCatchSwitch<'a>( Builder: &Builder<'a>, ParentPad: Option<&'a Value>, - BB: Option<&'a BasicBlock>, + UnwindBB: Option<&'a BasicBlock>, NumHandlers: c_uint, Name: *const c_char, ) -> Option<&'a Value>; - pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMAddHandler<'a>(CatchSwitch: &'a Value, Dest: &'a BasicBlock); pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); // Add a case to the switch instruction @@ -1614,11 +1640,12 @@ extern "C" { DestTy: &'a Type, Name: *const c_char, ) -> &'a Value; - pub fn LLVMRustBuildIntCast<'a>( + pub fn LLVMBuildIntCast2<'a>( B: &Builder<'a>, Val: &'a Value, DestTy: &'a Type, - IsSigned: bool, + IsSigned: Bool, + Name: *const c_char, ) -> &'a Value; // Comparisons @@ -1907,7 +1934,7 @@ extern "C" { ); pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; @@ -1965,6 +1992,21 @@ extern "C" { Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateMethod<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + Flags: DIFlags, + SPFlags: DISPFlags, + TParam: &'a DIArray, + ) -> &'a DISubprogram; + pub fn LLVMRustDIBuilderCreateBasicType<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2227,7 +2269,7 @@ extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine, cpu: *const c_char); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2256,6 +2298,7 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, + ForceEmulatedTls: bool, ) -> Option<&'static mut TargetMachine>; pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); pub fn LLVMRustAddLibraryInfo<'a>( @@ -2442,12 +2485,6 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; - pub fn LLVMRustThinLTOGetDICompileUnit( - M: &Module, - CU1: &mut *mut c_void, - CU2: &mut *mut c_void, - ); - pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index f820e752371..4f5cc575da6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -137,6 +137,7 @@ impl FromStr for ArchiveKind { "bsd" => Ok(ArchiveKind::K_BSD), "darwin" => Ok(ArchiveKind::K_DARWIN), "coff" => Ok(ArchiveKind::K_COFF), + "aix_big" => Ok(ArchiveKind::K_AIXBIG), _ => Err(()), } } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 46692fd5e8b..03be0654b50 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -16,7 +16,6 @@ use rustc_session::config::PrintRequest; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use smallvec::{smallvec, SmallVec}; use std::ffi::{CStr, CString}; use std::path::Path; @@ -132,6 +131,60 @@ pub fn time_trace_profiler_finish(file_name: &Path) { } } +pub enum TargetFeatureFoldStrength<'a> { + // The feature is only tied when enabling the feature, disabling + // this feature shouldn't disable the tied feature. + EnableOnly(&'a str), + // The feature is tied for both enabling and disabling this feature. + Both(&'a str), +} + +impl<'a> TargetFeatureFoldStrength<'a> { + fn as_str(&self) -> &'a str { + match self { + TargetFeatureFoldStrength::EnableOnly(feat) => feat, + TargetFeatureFoldStrength::Both(feat) => feat, + } + } +} + +pub struct LLVMFeature<'a> { + pub llvm_feature_name: &'a str, + pub dependency: Option<TargetFeatureFoldStrength<'a>>, +} + +impl<'a> LLVMFeature<'a> { + pub fn new(llvm_feature_name: &'a str) -> Self { + Self { llvm_feature_name, dependency: None } + } + + pub fn with_dependency( + llvm_feature_name: &'a str, + dependency: TargetFeatureFoldStrength<'a>, + ) -> Self { + Self { llvm_feature_name, dependency: Some(dependency) } + } + + pub fn contains(&self, feat: &str) -> bool { + self.iter().any(|dep| dep == feat) + } + + pub fn iter(&'a self) -> impl Iterator<Item = &'a str> { + let dependencies = self.dependency.iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + +impl<'a> IntoIterator for LLVMFeature<'a> { + type Item = &'a str; + type IntoIter = impl Iterator<Item = &'a str>; + + fn into_iter(self) -> Self::IntoIter { + let dependencies = self.dependency.into_iter().map(|feat| feat.as_str()); + std::iter::once(self.llvm_feature_name).chain(dependencies) + } +} + // WARNING: the features after applying `to_llvm_features` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. @@ -147,42 +200,65 @@ pub fn time_trace_profiler_finish(file_name: &Path) { // Though note that Rust can also be build with an external precompiled version of LLVM // which might lead to failures if the oldest tested / supported LLVM version // doesn't yet support the relevant intrinsics -pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { - ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"], - ("x86", "pclmulqdq") => smallvec!["pclmul"], - ("x86", "rdrand") => smallvec!["rdrnd"], - ("x86", "bmi1") => smallvec!["bmi"], - ("x86", "cmpxchg16b") => smallvec!["cx16"], - // FIXME: These aliases are misleading, and should be removed before avx512_target_feature is - // stabilized. They must remain until std::arch switches off them. - // rust#100752 - ("x86", "avx512vaes") => smallvec!["vaes"], - ("x86", "avx512gfni") => smallvec!["gfni"], - ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], - ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], - ("aarch64", "dpb") => smallvec!["ccpp"], - ("aarch64", "dpb2") => smallvec!["ccdp"], - ("aarch64", "frintts") => smallvec!["fptoint"], - ("aarch64", "fcma") => smallvec!["complxnum"], - ("aarch64", "pmuv3") => smallvec!["perfmon"], - ("aarch64", "paca") => smallvec!["pauth"], - ("aarch64", "pacg") => smallvec!["pauth"], - // Rust ties fp and neon together. In LLVM neon implicitly enables fp, - // but we manually enable neon when a feature only implicitly enables fp - ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], - ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], - ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], - ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], - ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], - ("aarch64", "sve") => smallvec!["sve", "neon"], - ("aarch64", "sve2") => smallvec!["sve2", "neon"], - ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], - ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], - ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], - ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], - (_, s) => smallvec![s], + ("x86", "sse4.2") => { + LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32")) + } + ("x86", "pclmulqdq") => LLVMFeature::new("pclmul"), + ("x86", "rdrand") => LLVMFeature::new("rdrnd"), + ("x86", "bmi1") => LLVMFeature::new("bmi"), + ("x86", "cmpxchg16b") => LLVMFeature::new("cx16"), + ("aarch64", "rcpc2") => LLVMFeature::new("rcpc-immo"), + ("aarch64", "dpb") => LLVMFeature::new("ccpp"), + ("aarch64", "dpb2") => LLVMFeature::new("ccdp"), + ("aarch64", "frintts") => LLVMFeature::new("fptoint"), + ("aarch64", "fcma") => LLVMFeature::new("complxnum"), + ("aarch64", "pmuv3") => LLVMFeature::new("perfmon"), + ("aarch64", "paca") => LLVMFeature::new("pauth"), + ("aarch64", "pacg") => LLVMFeature::new("pauth"), + // Rust ties fp and neon together. + ("aarch64", "neon") => { + LLVMFeature::with_dependency("neon", TargetFeatureFoldStrength::Both("fp-armv8")) + } + // In LLVM neon implicitly enables fp, but we manually enable + // neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => { + LLVMFeature::with_dependency("f32mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "f64mm") => { + LLVMFeature::with_dependency("f64mm", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fhm") => { + LLVMFeature::with_dependency("fp16fml", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "fp16") => { + LLVMFeature::with_dependency("fullfp16", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "jsconv") => { + LLVMFeature::with_dependency("jsconv", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve") => { + LLVMFeature::with_dependency("sve", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2") => { + LLVMFeature::with_dependency("sve2", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-aes") => { + LLVMFeature::with_dependency("sve2-aes", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sm4") => { + LLVMFeature::with_dependency("sve2-sm4", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-sha3") => { + LLVMFeature::with_dependency("sve2-sha3", TargetFeatureFoldStrength::EnableOnly("neon")) + } + ("aarch64", "sve2-bitperm") => LLVMFeature::with_dependency( + "sve2-bitperm", + TargetFeatureFoldStrength::EnableOnly("neon"), + ), + (_, s) => LLVMFeature::new(s), } } @@ -280,18 +356,17 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { let mut rustc_target_features = supported_target_features(sess) .iter() .map(|(feature, _gate)| { - let desc = if let Some(llvm_feature) = to_llvm_features(sess, *feature).first() { - // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + let llvm_feature = to_llvm_features(sess, *feature).llvm_feature_name; + let desc = match llvm_target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok() { Some(index) => { known_llvm_target_features.insert(llvm_feature); llvm_target_features[index].1 } None => "", - } - } else { - "" - }; + }; + (*feature, desc) }) .collect::<Vec<_>>(); @@ -329,7 +404,14 @@ pub(crate) fn print(req: PrintRequest, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); match req { - PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetCPUs => { + // SAFETY generate a C compatible string from a byte slice to pass + // the target CPU name into LLVM, the lifetime of the reference is + // at least as long as the C function + let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) + .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); + unsafe { llvm::LLVMRustPrintTargetCPUs(tm, cpu_cstring.as_ptr()) }; + } PrintRequest::TargetFeatures => print_target_features(sess, tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } @@ -468,10 +550,19 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str // passing requests down to LLVM. This means that all in-language // features also work on the command line instead of having two // different names when the LLVM name and the Rust name differ. + let llvm_feature = to_llvm_features(sess, feature); + Some( - to_llvm_features(sess, feature) - .into_iter() - .map(move |f| format!("{}{}", enable_disable, f)), + std::iter::once(format!("{}{}", enable_disable, llvm_feature.llvm_feature_name)) + .chain(llvm_feature.dependency.into_iter().filter_map(move |feat| { + match (enable_disable, feat) { + ('-' | '+', TargetFeatureFoldStrength::Both(f)) + | ('+', TargetFeatureFoldStrength::EnableOnly(f)) => { + Some(format!("{}{}", enable_disable, f)) + } + _ => None, + } + })), ) }) .flatten(); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index d0ae36349df..c24854b27a0 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -48,10 +48,10 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); + let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); @@ -125,8 +125,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) - .unwrap_or(false); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index ff111d96f84..d3fad5699c8 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -53,7 +53,7 @@ impl<'ll> CodegenCx<'ll, '_> { } pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) } } ///x Creates an integer type with the given number of bits, e.g., i24 @@ -291,8 +291,24 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn add_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { - let typeid_metadata = self.typeid_metadata(typeid); + let typeid_metadata = self.typeid_metadata(typeid).unwrap(); let v = [self.const_usize(0), typeid_metadata]; unsafe { llvm::LLVMGlobalSetMetadata( @@ -307,13 +323,28 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - fn typeid_metadata(&self, typeid: String) -> &'ll Value { - unsafe { + fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> { + Some(unsafe { llvm::LLVMMDStringInContext( self.llcx, typeid.as_ptr() as *const c_char, typeid.len() as c_uint, ) + }) + } + + fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { + let kcfi_type_metadata = self.const_u32(kcfi_typeid); + unsafe { + llvm::LLVMRustGlobalAddMetadata( + function, + llvm::MD_kcfi_type as c_uint, + llvm::LLVMMDNodeInContext2( + self.llcx, + &llvm::LLVMValueAsMetadata(kcfi_type_metadata), + 1, + ), + ) } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index e264ce78f0d..a493c9c0548 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -198,8 +198,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { match self.abi { Abi::Scalar(_) | Abi::Vector { .. } => true, - Abi::ScalarPair(..) => false, - Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), + Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, } } |