diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src/back/write.rs')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 428 |
1 files changed, 113 insertions, 315 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 2b465ce40e7..53b4296802e 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,7 +5,10 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::consts; -use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::errors::{ + CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode, +}; +use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; @@ -21,15 +24,14 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; -use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; -use tracing::debug; +use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; use std::ffi::CString; use std::fs; @@ -39,10 +41,10 @@ use std::slice; use std::str; use std::sync::Arc; -pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { +pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError { match llvm::last_error() { - Some(err) => handler.fatal(&format!("{}: {}", msg, err)), - None => handler.fatal(msg), + Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)), + None => handler.emit_almost_fatal(err), } } @@ -87,10 +89,9 @@ pub fn write_output_file<'ll>( } } - result.into_result().map_err(|()| { - let msg = format!("could not write output to {}", output.display()); - llvm_err(handler, &msg) - }) + result + .into_result() + .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output })) } } @@ -100,7 +101,7 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm: // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise()) } pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { @@ -119,7 +120,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise()) } pub fn to_llvm_opt_settings( @@ -205,7 +206,7 @@ pub fn target_machine_factory( sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; - let asm_comments = sess.asm_comments(); + let asm_comments = sess.opts.unstable_opts.asm_comments; let relax_elf_relocations = sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); @@ -214,6 +215,8 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); + let force_emulated_tls = sess.target.force_emulated_tls; + Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; @@ -239,12 +242,11 @@ pub fn target_machine_factory( relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), + force_emulated_tls, ) }; - tm.ok_or_else(|| { - format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) - }) + tm.ok_or_else(|| LlvmError::CreateTargetMachine { triple: triple.clone() }) }) } @@ -304,7 +306,6 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes.as_ptr(), remark_passes.len(), ); - llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); DiagnosticHandlers { data, llcx, old_handler } } } @@ -312,9 +313,7 @@ impl<'a> DiagnosticHandlers<'a> { impl<'a> Drop for DiagnosticHandlers<'a> { fn drop(&mut self) { - use std::ptr::null_mut; unsafe { - llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); drop(Box::from_raw(self.data)); } @@ -342,16 +341,6 @@ fn report_inline_asm( cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); } -unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { - if user.is_null() { - return; - } - let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); - - let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); - report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); -} - unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { if user.is_null() { return; @@ -370,25 +359,37 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void }; if enabled { - diag_handler.note_without_error(&format!( - "{}:{}:{}: {}: {}", - opt.filename, opt.line, opt.column, opt.pass_name, opt.message, - )); + diag_handler.emit_note(FromLlvmOptimizationDiag { + filename: &opt.filename, + line: opt.line, + column: opt.column, + pass_name: &opt.pass_name, + kind: match opt.kind { + OptimizationDiagnosticKind::OptimizationRemark => "success", + OptimizationDiagnosticKind::OptimizationMissed + | OptimizationDiagnosticKind::OptimizationFailure => "missed", + OptimizationDiagnosticKind::OptimizationAnalysis + | OptimizationDiagnosticKind::OptimizationAnalysisFPCommute + | OptimizationDiagnosticKind::OptimizationAnalysisAliasing => "analysis", + OptimizationDiagnosticKind::OptimizationRemarkOther => "other", + }, + message: &opt.message, + }); } } llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.warn(&msg); + diag_handler.emit_warning(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { - let msg = llvm::build_string(|s| { + let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.err(&msg); + diag_handler.emit_err(FromLlvmDiag { message }); } llvm::diagnostic::UnknownDiagnostic(..) => {} } @@ -423,7 +424,11 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } -pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( +fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { + config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap()) +} + +pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, module: &ModuleCodegen<ModuleLlvm>, @@ -438,6 +443,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + let instr_profile_output_path = get_instr_profile_output_path(config); // Sanitizer instrumentation is only inserted during the pre-link optimization stage. let sanitizer_options = if !is_lto { Some(llvm::SanitizerOptions { @@ -449,16 +455,19 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), + sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS), + sanitize_kernel_address_recover: config + .sanitizer_recover + .contains(SanitizerSet::KERNELADDRESS), }) } else { None }; - let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { - Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) - } else { - None - }; + let mut llvm_profiler = cgcx + .prof + .llvm_recording_enabled() + .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())); let llvm_selfprofiler = llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); @@ -470,7 +479,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. - let result = llvm::LLVMRustOptimizeWithNewPassManager( + let result = llvm::LLVMRustOptimize( module.module_llvm.llmod(), &*module.module_llvm.tm, to_pass_builder_opt_level(opt_level), @@ -488,6 +497,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, + instr_profile_output_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_gcov, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, @@ -499,7 +509,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( llvm_plugins.as_ptr().cast(), llvm_plugins.len(), ); - result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) + result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -513,18 +523,11 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let tm = &*module.module_llvm.tm; let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { - diag_handler.warn( - "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", - ); - } - if config.emit_no_opt_bc { let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); let out = path_to_c_string(&out); @@ -532,191 +535,24 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if llvm_util::should_use_new_llvm_pass_manager( - &config.new_llvm_pass_manager, - &cgcx.target_arch, - ) { - let opt_stage = match cgcx.lto { - Lto::Fat => llvm::OptStage::PreLinkFatLTO, - Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, - _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, - _ => llvm::OptStage::PreLinkNoLTO, - }; - return optimize_with_new_llvm_pass_manager( - cgcx, - diag_handler, - module, - config, - opt_level, - opt_stage, - ); - } - - if cgcx.prof.llvm_recording_enabled() { - diag_handler - .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); - } - - // Create the two optimizing pass managers. These mirror what clang - // does, and are by populated by LLVM's default PassManagerBuilder. - // Each manager has a different set of passes, but they also share - // some common passes. - let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); - let mpm = llvm::LLVMCreatePassManager(); - - { - let find_pass = |pass_name: &str| { - let pass_name = SmallCStr::new(pass_name); - llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) - }; - - if config.verify_llvm_ir { - // Verification should run as the very first pass. - llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); - } - - let mut extra_passes = Vec::new(); - let mut have_name_anon_globals_pass = false; - - for pass_name in &config.passes { - if pass_name == "lint" { - // Linting should also be performed early, directly on the generated IR. - llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); - continue; - } - - if let Some(pass) = find_pass(pass_name) { - extra_passes.push(pass); - } else { - diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); - } - - if pass_name == "name-anon-globals" { - have_name_anon_globals_pass = true; - } - } - - // Instrumentation must be inserted before optimization, - // otherwise LLVM may optimize some functions away which - // breaks llvm-cov. - // - // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. - if config.instrument_gcov { - llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); - } - if config.instrument_coverage { - llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); - } - if config.debug_info_for_profiling { - llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); - } - - add_sanitizer_passes(config, &mut extra_passes); - - // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need - // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise - // we'll get errors in LLVM. - let using_thin_buffers = config.bitcode_needed(); - if !config.no_prepopulate_passes { - llvm::LLVMAddAnalysisPasses(tm, fpm); - llvm::LLVMAddAnalysisPasses(tm, mpm); - let opt_level = to_llvm_opt_settings(opt_level).0; - let prepare_for_thin_lto = cgcx.lto == Lto::Thin - || cgcx.lto == Lto::ThinLocal - || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); - with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { - llvm::LLVMRustAddLastExtensionPasses( - b, - extra_passes.as_ptr(), - extra_passes.len() as size_t, - ); - llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); - llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); - }); - - have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; - if using_thin_buffers && !prepare_for_thin_lto { - llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); - have_name_anon_globals_pass = true; - } - } else { - // If we don't use the standard pipeline, directly populate the MPM - // with the extra passes. - for pass in extra_passes { - llvm::LLVMRustAddPass(mpm, pass); - } - } - - if using_thin_buffers && !have_name_anon_globals_pass { - // As described above, this will probably cause an error in LLVM - if config.no_prepopulate_passes { - diag_handler.err( - "The current compilation is going to use thin LTO buffers \ - without running LLVM's NameAnonGlobals pass. \ - This will likely cause errors in LLVM. Consider adding \ - -C passes=name-anon-globals to the compiler command line.", - ); - } else { - bug!( - "We are using thin LTO buffers without running the NameAnonGlobals pass. \ - This will likely cause errors in LLVM and should never happen." - ); - } - } - } - - diag_handler.abort_if_errors(); - - // Finally, run the actual optimization passes - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_function_passes", - &*module.name, - ); - llvm::LLVMRustRunFunctionPassManager(fpm, llmod); - } - { - let _timer = cgcx.prof.extra_verbose_generic_activity( - "LLVM_module_optimize_module_passes", - &*module.name, - ); - llvm::LLVMRunPassManager(mpm, llmod); - } - - // Deallocate managers that we're now done with - llvm::LLVMDisposePassManager(fpm); - llvm::LLVMDisposePassManager(mpm); + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + return llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage); } Ok(()) } -unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { - if config.sanitizer.contains(SanitizerSet::ADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); - passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); - passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); - } - if config.sanitizer.contains(SanitizerSet::MEMORY) { - let track_origins = config.sanitizer_memory_track_origins as c_int; - let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); - passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); - } - if config.sanitizer.contains(SanitizerSet::THREAD) { - passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); - } - if config.sanitizer.contains(SanitizerSet::HWADDRESS) { - let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); - passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); - } -} - pub(crate) fn link( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, mut modules: Vec<ModuleCodegen<ModuleLlvm>>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { use super::lto::{Linker, ModuleBuffer}; - // Sort the modules by name to ensure to ensure deterministic behavior. + // Sort the modules by name to ensure deterministic behavior. modules.sort_by(|a, b| a.name.cmp(&b.name)); let (first, elements) = modules.split_first().expect("Bug! modules must contain at least one module."); @@ -726,8 +562,7 @@ pub(crate) fn link( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); linker.add(buffer.data()).map_err(|()| { - let msg = format!("failed to serialize module {:?}", module.name); - llvm_err(diag_handler, &msg) + llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name }) })?; } drop(linker); @@ -790,7 +625,7 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); let data = thin.data(); if let Some(bitcode_filename) = bc_out.file_name() { @@ -805,9 +640,8 @@ pub(crate) unsafe fn codegen( let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); - if let Err(e) = fs::write(&bc_out, data) { - let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); - diag_handler.err(&msg); + if let Err(err) = fs::write(&bc_out, data) { + diag_handler.emit_err(WriteBytecode { path: &bc_out, err }); } } @@ -857,10 +691,9 @@ pub(crate) unsafe fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| { - let msg = format!("failed to write LLVM IR to {}", out.display()); - llvm_err(diag_handler, &msg) - })?; + result + .into_result() + .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?; } if config.emit_asm { @@ -928,8 +761,8 @@ pub(crate) unsafe fn codegen( EmitObj::Bitcode => { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); - if let Err(e) = link_or_copy(&bc_out, &obj_out) { - diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); + if let Err(err) = link_or_copy(&bc_out, &obj_out) { + diag_handler.emit_err(CopyBitcode { err }); } if !config.emit_bc { @@ -941,14 +774,25 @@ pub(crate) unsafe fn codegen( EmitObj::None => {} } + record_llvm_cgu_instructions_stats(&cgcx.prof, llmod); drop(handlers); } + // `.dwo` files are only emitted if: + // + // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not + // produce dwarf objects, even if otherwise enabled) + // - Target supports Split DWARF + // - Split debuginfo is enabled + // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different + // sections in the `.o` files). + let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_)) + && cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split; Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf - && cgcx.split_debuginfo != SplitDebuginfo::Off - && cgcx.split_dwarf_kind == SplitDwarfKind::Split, + dwarf_object_emitted, config.emit_bc, &cgcx.output_filenames, )) @@ -1047,11 +891,11 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.module\0".as_ptr().cast(), + c"rustc.embedded.module".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + let section = if is_apple { c"__LLVM,__bitcode" } else { c".llvmbc" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); @@ -1060,91 +904,25 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.cmdline\0".as_ptr().cast(), + c"rustc.embedded.cmdline".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + let section = if is_apple { c"__LLVM,__cmdline" } else { c".llvmcmd" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); - } -} - -pub unsafe fn with_llvm_pmb( - llmod: &llvm::Module, - config: &ModuleConfig, - opt_level: llvm::CodeGenOptLevel, - prepare_for_thin_lto: bool, - f: &mut dyn FnMut(&llvm::PassManagerBuilder), -) { - use std::ptr; - - // Create the PassManagerBuilder for LLVM. We configure it with - // reasonable defaults and prepare it to actually populate the pass - // manager. - let builder = llvm::LLVMRustPassManagerBuilderCreate(); - let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); - let inline_threshold = config.inline_threshold; - let pgo_gen_path = get_pgo_gen_path(config); - let pgo_use_path = get_pgo_use_path(config); - let pgo_sample_use_path = get_pgo_sample_use_path(config); - - llvm::LLVMRustConfigurePassManagerBuilder( - builder, - opt_level, - config.merge_functions, - config.vectorize_slp, - config.vectorize_loop, - prepare_for_thin_lto, - pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), - opt_size as c_int, - ); - - llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); - - // Here we match what clang does (kinda). For O0 we only inline - // always-inline functions (but don't add lifetime intrinsics), at O1 we - // inline with lifetime intrinsics, and O2+ we add an inliner with a - // thresholds copied from clang. - match (opt_level, opt_size, inline_threshold) { - (.., Some(t)) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); - } - (llvm::CodeGenOptLevel::Aggressive, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); - } - (_, llvm::CodeGenOptSizeDefault, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); - } - (_, llvm::CodeGenOptSizeAggressive, _) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); - } - (llvm::CodeGenOptLevel::None, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Less, ..) => { - llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); - } - (llvm::CodeGenOptLevel::Default, ..) => { - llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); - } + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } - - f(builder); - llvm::LLVMRustPassManagerBuilderDispose(builder); } // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs -// when using MSVC linker. We do this only for data, as linker can fix up +// when using MSVC linker. We do this only for data, as linker can fix up // code references on its own. // See #26591, #27438 fn create_msvc_imps( @@ -1210,3 +988,23 @@ fn record_artifact_size( self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); } } + +fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) { + if !prof.enabled() { + return; + } + + let raw_stats = + llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(&llmod, s) }) + .expect("cannot get module instruction stats"); + + #[derive(serde::Deserialize)] + struct InstructionsStats { + module: String, + total: u64, + } + + let InstructionsStats { module, total } = + serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats"); + prof.artifact_size("cgu_instructions", module, total); +} |