diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 43 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 47 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 34 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/base.rs | 28 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/consts.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs | 22 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/doc.md | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/utils.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 32 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 5 |
16 files changed, 192 insertions, 119 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 91d132eb343..e994001f96f 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -312,11 +312,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -impl AsmMethods for CodegenCx<'_, '_> { +impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { fn codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], - operands: &[GlobalAsmOperandRef], + operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], ) { @@ -342,6 +342,29 @@ impl AsmMethods for CodegenCx<'_, '_> { // here unlike normal inline assembly. template_str.push_str(string); } + GlobalAsmOperandRef::SymFn { instance } => { + let llval = self.get_fn(instance); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + let llval = self + .renamed_statics + .borrow() + .get(&def_id) + .copied() + .unwrap_or_else(|| self.get_static(def_id)); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } } } } @@ -579,7 +602,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", InlineAsmRegClass::X86( - X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg, + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0, ) => unreachable!("clobber-only"), InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", @@ -664,7 +689,11 @@ fn modifier_to_llvm( _ => unreachable!(), }, InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, - InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg) => { + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, @@ -734,7 +763,11 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), - InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg) => { + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 0f5b1c08ec2..d0724baf9e0 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -6,9 +6,7 @@ use crate::llvm::{self, build_string, False, True}; use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; -use rustc_codegen_ssa::back::write::{ - CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, -}; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; @@ -16,7 +14,7 @@ use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; -use rustc_middle::middle::exported_symbols::SymbolExportLevel; +use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; use tracing::{debug, info}; @@ -55,8 +53,8 @@ fn prepare_lto( Lto::No => panic!("didn't request LTO but we're doing LTO"), }; - let symbol_filter = &|&(ref name, level): &(String, SymbolExportLevel)| { - if level.is_below_threshold(export_threshold) { + let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { + if info.level.is_below_threshold(export_threshold) || info.used { Some(CString::new(name.as_str()).unwrap()) } else { None @@ -313,7 +311,9 @@ fn fat_lto( for (bc_decoded, name) in serialized_modules { let _timer = cgcx .prof - .generic_activity_with_arg("LLVM_fat_lto_link_module", format!("{:?}", name)); + .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { + recorder.record_arg(format!("{:?}", name)) + }); info!("linking {:?}", name); let data = bc_decoded.data(); linker.add(data).map_err(|()| { @@ -351,7 +351,7 @@ fn fat_lto( } } - Ok(LtoModuleCodegen::Fat { module: Some(module), _serialized_bitcode: serialized_bitcode }) + Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) } crate struct Linker<'a>(&'a mut llvm::Linker<'a>); @@ -394,15 +394,15 @@ impl Drop for Linker<'_> { /// /// At a high level Thin LTO looks like: /// -/// 1. Prepare a "summary" of each LLVM module in question which describes -/// the values inside, cost of the values, etc. -/// 2. Merge the summaries of all modules in question into one "index" -/// 3. Perform some global analysis on this index -/// 4. For each module, use the index and analysis calculated previously to -/// perform local transformations on the module, for example inlining -/// small functions from other modules. -/// 5. Run thin-specific optimization passes over each module, and then code -/// generate everything at the end. +/// 1. Prepare a "summary" of each LLVM module in question which describes +/// the values inside, cost of the values, etc. +/// 2. Merge the summaries of all modules in question into one "index" +/// 3. Perform some global analysis on this index +/// 4. For each module, use the index and analysis calculated previously to +/// perform local transformations on the module, for example inlining +/// small functions from other modules. +/// 5. Run thin-specific optimization passes over each module, and then code +/// generate everything at the end. /// /// The summary for each module is intended to be quite cheap, and the global /// index is relatively quite cheap to create as well. As a result, the goal of @@ -576,11 +576,11 @@ fn thin_lto( pub(crate) fn run_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, - module: &ModuleCodegen<ModuleLlvm>, - config: &ModuleConfig, + module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let config = cgcx.config(module.kind); // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -623,7 +623,7 @@ pub(crate) fn run_pass_manager( if thin { llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); } else { - llvm::LLVMPassManagerBuilderPopulateLTOPassManager( + llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( b, pm, /* Internalize = */ False, /* RunInliner = */ True, ); } @@ -724,7 +724,7 @@ impl Drop for ThinBuffer { } pub unsafe fn optimize_thin_module( - thin_module: &mut ThinModule<LlvmCodegenBackend>, + thin_module: ThinModule<LlvmCodegenBackend>, cgcx: &CodegenContext<LlvmCodegenBackend>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { let diag_handler = cgcx.create_diag_handler(); @@ -741,7 +741,7 @@ pub unsafe fn optimize_thin_module( // that LLVM Context and Module. let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; - let module = ModuleCodegen { + let mut module = ModuleCodegen { module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, name: thin_module.name().to_string(), kind: ModuleKind::Regular, @@ -857,8 +857,7 @@ pub unsafe fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - let config = cgcx.config(module.kind); - run_pass_manager(cgcx, &diag_handler, &module, config, true)?; + run_pass_manager(cgcx, &diag_handler, &mut module, true)?; save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c18719d4ad7..99e30531c22 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -523,6 +523,12 @@ pub(crate) unsafe fn optimize( let module_name = module.name.clone(); let module_name = Some(&module_name[..]); + if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { + diag_handler.warn( + "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", + ); + } + if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); @@ -628,8 +634,8 @@ pub(crate) unsafe fn optimize( extra_passes.as_ptr(), extra_passes.len() as size_t, ); - llvm::LLVMPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMPassManagerBuilderPopulateModulePassManager(b, mpm); + llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); + llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); }); have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; @@ -721,8 +727,7 @@ pub(crate) fn link( let mut linker = Linker::new(first.module_llvm.llmod()); for module in elements { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_link_module", format!("{:?}", module.name)); + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { let msg = format!("failed to serialize module {:?}", module.name); @@ -1086,7 +1091,7 @@ pub unsafe fn with_llvm_pmb( // Create the PassManagerBuilder for LLVM. We configure it with // reasonable defaults and prepare it to actually populate the pass // manager. - let builder = llvm::LLVMPassManagerBuilderCreate(); + let builder = llvm::LLVMRustPassManagerBuilderCreate(); let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); let inline_threshold = config.inline_threshold; let pgo_gen_path = get_pgo_gen_path(config); @@ -1103,14 +1108,9 @@ pub unsafe fn with_llvm_pmb( pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + opt_size as c_int, ); - llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32); - - if opt_size != llvm::CodeGenOptSizeNone { - llvm::LLVMPassManagerBuilderSetDisableUnrollLoops(builder, 1); - } - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); // Here we match what clang does (kinda). For O0 we only inline @@ -1119,16 +1119,16 @@ pub unsafe fn with_llvm_pmb( // thresholds copied from clang. match (opt_level, opt_size, inline_threshold) { (.., Some(t)) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); } (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); } (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 75); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); } (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 25); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); } (llvm::CodeGenOptLevel::None, ..) => { llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); @@ -1137,12 +1137,12 @@ pub unsafe fn with_llvm_pmb( llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); } (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 225); + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); } } f(builder); - llvm::LLVMPassManagerBuilderDispose(builder); + llvm::LLVMRustPassManagerBuilderDispose(builder); } // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index dd3ada44389..86f92dc0239 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -74,10 +74,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen<ModuleLlvm> { let cgu = tcx.codegen_unit(cgu_name); - let _prof_timer = tcx.prof.generic_activity_with_args( - "codegen_module", - &[cgu_name.to_string(), cgu.size_estimate().to_string()], - ); + let _prof_timer = + tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { + recorder.record_arg(cgu_name.to_string()); + recorder.record_arg(cgu.size_estimate().to_string()); + }); // Instantiate monomorphizations without filling out definitions yet... let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { @@ -99,15 +100,6 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } - // Run replace-all-uses-with for statics that need it - for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { - unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); - llvm::LLVMDeleteGlobal(old_g); - } - } - // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { @@ -122,6 +114,16 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen cx.create_compiler_used_variable() } + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + // Finalize debuginfo if cx.sess().opts.debuginfo != DebugInfo::None { cx.debuginfo_finalize(); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index cb4073528e1..4d3f3f318b8 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -412,6 +412,13 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { llvm::LLVMRustSetLinkage(new_g, linkage); llvm::LLVMRustSetVisibility(new_g, visibility); + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + // To avoid breaking any invariants, we leave around the old // global for the moment; we'll replace all references to it // with the new global later. (See base::codegen_backend.) diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 98cf873ebbd..d296ee3b42c 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -14,6 +14,7 @@ use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; use rustc_middle::mir::mono::CodegenUnit; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, @@ -105,6 +106,12 @@ pub struct CodegenCx<'ll, 'tcx> { /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell<usize>, + + /// `codegen_static` will sometimes create a second global variable with a + /// different type and clear the symbol name of the original global. + /// `global_asm!` needs to be able to find this new global so that it can + /// compute the correct mangled symbol name to insert into the asm. + pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>, } pub struct TypeLowering<'ll> { @@ -436,6 +443,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { rust_try_fn: Cell::new(None), intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), + renamed_statics: Default::default(), } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 76caa3ceaaf..99e4ded62f1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -20,7 +20,6 @@ pub fn compute_mir_scopes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - fn_dbg_scope: &'ll DIScope, debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, ) { // Find all scopes with variables defined in them. @@ -38,40 +37,41 @@ pub fn compute_mir_scopes<'ll, 'tcx>( // Nothing to emit, of course. None }; - + let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, mir, fn_dbg_scope, &variables, debug_context, scope); + make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); } + assert!(instantiated.count() == mir.source_scopes.len()); } fn make_mir_scope<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - fn_dbg_scope: &'ll DIScope, variables: &Option<BitSet<SourceScope>>, debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + instantiated: &mut BitSet<SourceScope>, scope: SourceScope, ) { - if debug_context.scopes[scope].dbg_scope.is_some() { + if instantiated.contains(scope) { return; } let scope_data = &mir.source_scopes[scope]; let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, instance, mir, fn_dbg_scope, variables, debug_context, parent); + make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); debug_context.scopes[parent] } else { // The root is the function itself. let loc = cx.lookup_debug_loc(mir.span.lo()); debug_context.scopes[scope] = DebugScope { - dbg_scope: Some(fn_dbg_scope), - inlined_at: None, file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, + ..debug_context.scopes[scope] }; + instantiated.insert(scope); return; }; @@ -79,6 +79,7 @@ fn make_mir_scope<'ll, 'tcx>( // Do not create a DIScope if there are no variables defined in this // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. debug_context.scopes[scope] = parent_scope; + instantiated.insert(scope); return; } @@ -100,7 +101,7 @@ fn make_mir_scope<'ll, 'tcx>( None => unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlock( DIB(cx), - parent_scope.dbg_scope.unwrap(), + parent_scope.dbg_scope, file_metadata, loc.line, loc.col, @@ -116,9 +117,10 @@ fn make_mir_scope<'ll, 'tcx>( }); debug_context.scopes[scope] = DebugScope { - dbg_scope: Some(dbg_scope), + dbg_scope, inlined_at: inlined_at.or(parent_scope.inlined_at), file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, }; + instantiated.insert(scope); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md index 5a8976c6166..aaec4e68c17 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md @@ -27,9 +27,9 @@ The module is thus driven from an outside client with functions like Internally the module will try to reuse already created metadata by utilizing a cache. The way to get a shared metadata node when needed is thus to just call the corresponding function in this module: - - let file_metadata = file_metadata(cx, file); - +```ignore (illustrative) +let file_metadata = file_metadata(cx, file); +``` The function will take care of probing the cache for an existing node for that exact file path. @@ -63,7 +63,7 @@ struct List { will generate the following callstack with a naive DFS algorithm: -``` +```ignore (illustrative) describe(t = List) describe(t = i32) describe(t = Option<Box<List>>) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 74e194750fa..f2cf3b1ef5c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -437,11 +437,9 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { - DINodeCreationResult::new(build_basic_type_di_node(cx, t), false) - } - ty::Tuple(elements) if elements.is_empty() => { - DINodeCreationResult::new(build_basic_type_di_node(cx, t), false) + build_basic_type_di_node(cx, t) } + ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), @@ -640,7 +638,10 @@ impl MsvcBasicName for ty::FloatTy { } } -fn build_basic_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { +fn build_basic_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { debug!("build_basic_type_di_node: {:?}", t); // When targeting MSVC, emit MSVC style type names for compatibility with @@ -649,7 +650,13 @@ fn build_basic_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) - let (name, encoding) = match t.kind() { ty::Never => ("!", DW_ATE_unsigned), - ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => { + if cpp_like_debuginfo { + return build_tuple_type_di_node(cx, UniqueTypeId::for_ty(cx.tcx, t)); + } else { + ("()", DW_ATE_unsigned) + } + } ty::Bool => ("bool", DW_ATE_boolean), ty::Char => ("char", DW_ATE_UTF), ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), @@ -672,14 +679,14 @@ fn build_basic_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) - }; if !cpp_like_debuginfo { - return ty_di_node; + return DINodeCreationResult::new(ty_di_node, false); } let typedef_name = match t.kind() { ty::Int(int_ty) => int_ty.name_str(), ty::Uint(uint_ty) => uint_ty.name_str(), ty::Float(float_ty) => float_ty.name_str(), - _ => return ty_di_node, + _ => return DINodeCreationResult::new(ty_di_node, false), }; let typedef_di_node = unsafe { @@ -694,7 +701,7 @@ fn build_basic_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) - ) }; - typedef_di_node + DINodeCreationResult::new(typedef_di_node, false) } fn build_foreign_type_di_node<'ll, 'tcx>( diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 31bb9ed3185..d6e2c8ccdf4 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -372,7 +372,6 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range - .clone() .map(|variant_index| { let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( cx, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 4e6d3f88e67..6a164557a47 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -286,9 +286,8 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } // Initialize fn debug context (including scopes). - // FIXME(eddyb) figure out a way to not need `Option` for `dbg_scope`. let empty_scope = DebugScope { - dbg_scope: None, + dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), inlined_at: None, file_start_pos: BytePos(0), file_end_pos: BytePos(0), @@ -297,13 +296,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; // Fill in all the scopes, with the information from the MIR body. - compute_mir_scopes( - self, - instance, - mir, - self.dbg_scope_fn(instance, fn_abi, Some(llfn)), - &mut fn_debug_context, - ); + compute_mir_scopes(self, instance, mir, &mut fn_debug_context); Some(fn_debug_context) } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index fe9851cfa56..8f243673907 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -46,7 +46,7 @@ pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { } pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { - item_namespace(cx, cx.tcx.parent(def_id).expect("get_namespace_for_item: missing parent?")) + item_namespace(cx, cx.tcx.parent(def_id)) } #[derive(Debug, PartialEq, Eq)] diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 48840c76cac..cf9cf1b70aa 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1839,6 +1839,27 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, simd_neg: Int => neg, Float => fneg; } + if name == sym::simd_arith_offset { + // This also checks that the first operand is a ptr type. + let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { + span_bug!(span, "must be called with a vector of pointer types as first argument") + }); + let layout = bx.layout_of(pointee.ty); + let ptrs = args[0].immediate(); + // The second argument must be a ptr-sized integer. + // (We don't care about the signedness, this is wrapping anyway.) + let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); + if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { + span_bug!( + span, + "must be called with a vector of pointer-sized integers as second argument" + ); + } + let offsets = args[1].immediate(); + + return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); + } + if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { let lhs = args[0].immediate(); let rhs = args[1].immediate(); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 3152c505af0..0bead4629a6 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,7 +5,6 @@ //! This API is completely unstable and subject to change. #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] -#![feature(bool_to_option)] #![feature(crate_visibility_modifier)] #![feature(let_chains)] #![feature(let_else)] @@ -104,19 +103,18 @@ impl Drop for TimeTraceProfiler { } impl ExtraBackendMethods for LlvmCodegenBackend { - fn new_metadata(&self, tcx: TyCtxt<'_>, mod_name: &str) -> ModuleLlvm { - ModuleLlvm::new_metadata(tcx, mod_name) - } - fn codegen_allocator<'tcx>( &self, tcx: TyCtxt<'tcx>, - module_llvm: &mut ModuleLlvm, module_name: &str, kind: AllocatorKind, has_alloc_error_handler: bool, - ) { - unsafe { allocator::codegen(tcx, module_llvm, module_name, kind, has_alloc_error_handler) } + ) -> ModuleLlvm { + let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); + unsafe { + allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); + } + module_llvm } fn compile_codegen_unit( &self, @@ -210,9 +208,16 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> Result<(), FatalError> { back::write::optimize(cgcx, diag_handler, module, config) } + fn optimize_fat( + cgcx: &CodegenContext<Self>, + module: &mut ModuleCodegen<Self::Module>, + ) -> Result<(), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + back::lto::run_pass_manager(cgcx, &diag_handler, module, false) + } unsafe fn optimize_thin( cgcx: &CodegenContext<Self>, - thin: &mut ThinModule<Self>, + thin: ThinModule<Self>, ) -> Result<ModuleCodegen<Self::Module>, FatalError> { back::lto::optimize_thin_module(thin, cgcx) } @@ -230,15 +235,6 @@ impl WriteBackendMethods for LlvmCodegenBackend { fn serialize_module(module: ModuleCodegen<Self::Module>) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) } - fn run_lto_pass_manager( - cgcx: &CodegenContext<Self>, - module: &ModuleCodegen<Self::Module>, - config: &ModuleConfig, - thin: bool, - ) -> Result<(), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - back::lto::run_pass_manager(cgcx, &diag_handler, module, config, thin) - } } unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 375b9927c86..13baaddccd4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1825,24 +1825,22 @@ extern "C" { pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); - pub fn LLVMPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; - pub fn LLVMPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); - pub fn LLVMPassManagerBuilderSetSizeLevel(PMB: &PassManagerBuilder, Value: Bool); - pub fn LLVMPassManagerBuilderSetDisableUnrollLoops(PMB: &PassManagerBuilder, Value: Bool); - pub fn LLVMPassManagerBuilderUseInlinerWithThreshold( + pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; + pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); + pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( PMB: &PassManagerBuilder, threshold: c_uint, ); - pub fn LLVMPassManagerBuilderPopulateModulePassManager( + pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, ); - pub fn LLVMPassManagerBuilderPopulateFunctionPassManager( + pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, ); - pub fn LLVMPassManagerBuilderPopulateLTOPassManager( + pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( PMB: &PassManagerBuilder, PM: &PassManager<'_>, Internalize: Bool, @@ -2308,6 +2306,7 @@ extern "C" { PGOGenPath: *const c_char, PGOUsePath: *const c_char, PGOSampleUsePath: *const c_char, + SizeLevel: c_int, ); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, @@ -2537,4 +2536,6 @@ extern "C" { remark_passes_len: usize, ); + #[allow(improper_ctypes)] + pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index c24e369ae72..7b407c94e7b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -542,6 +542,11 @@ pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_a // The new pass manager is enabled by default for LLVM >= 13. // This matches Clang, which also enables it since Clang 13. + // Since LLVM 15, the legacy pass manager is no longer supported. + if llvm_util::get_version() >= (15, 0, 0) { + return true; + } + // There are some perf issues with the new pass manager when targeting // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. // See https://github.com/rust-lang/rust/issues/89609. |