diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
28 files changed, 572 insertions, 319 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b5f53f51838..0718bebb31b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -6,7 +6,7 @@ use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::MemFlags; @@ -15,7 +15,6 @@ use rustc_middle::ty::layout::LayoutOf; pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; use rustc_middle::ty::Ty; use rustc_session::config; -use rustc_target::abi::call::ArgAbi; pub use rustc_target::abi::call::*; use rustc_target::abi::{self, HasDataLayout, Int}; pub use rustc_target::spec::abi::Abi; @@ -254,7 +253,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { bx.lifetime_end(llscratch, scratch_size); } } else { - OperandValue::Immediate(val).store(bx, dst); + OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 798014d668e..ca376029735 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -76,7 +76,7 @@ pub(crate) unsafe fn codegen( // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.target.default_hidden_visibility { + if tcx.sess.default_hidden_visibility() { llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); } let val = tcx.sess.opts.unstable_opts.oom.should_panic(); @@ -85,7 +85,7 @@ pub(crate) unsafe fn codegen( let name = NO_ALLOC_SHIM_IS_UNSTABLE; let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - if tcx.sess.target.default_hidden_visibility { + if tcx.sess.default_hidden_visibility() { llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); } let llval = llvm::LLVMConstInt(i8, 0, False); @@ -130,11 +130,12 @@ fn create_wrapper_function( None }; - if tcx.sess.target.default_hidden_visibility { + if tcx.sess.default_hidden_visibility() { llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); } if tcx.sess.must_emit_unwind_tables() { - let uwtable = attributes::uwtable_attr(llcx); + let uwtable = + attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); } @@ -146,7 +147,7 @@ fn create_wrapper_function( } llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1323261ae92..a413466093b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -690,6 +690,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a", InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r", @@ -867,7 +868,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), - InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::S390x( + S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr, + ) => cx.type_i32(), InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(), diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 481741bb127..0a7ea599431 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -82,7 +82,7 @@ pub fn sanitize_attrs<'ll>( let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.sess.emit_err(SanitizerMemtagRequiresMte); + cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -95,11 +95,12 @@ pub fn sanitize_attrs<'ll>( /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. #[inline] -pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { +pub fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>) -> &Attribute { // NOTE: We should determine if we even need async unwind tables, as they // take have more overhead and if we can use sync unwind tables we // probably should. - llvm::CreateUWTableAttr(llcx, true) + let async_unwind = !use_sync_unwind.unwrap_or(false); + llvm::CreateUWTableAttr(llcx, async_unwind) } pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { @@ -333,7 +334,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // You can also find more info on why Windows always requires uwtables here: // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx)); + to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); } if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { @@ -444,7 +445,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( .next() .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); cx.tcx - .sess + .dcx() .create_err(TargetFeatureDisableOrEnable { features: f, span: Some(span), @@ -481,7 +482,7 @@ pub fn from_fn_attrs<'ll, 'tcx>( // `+multivalue` feature because the purpose of the wasm abi is to match // the WebAssembly specification, which has this feature. This won't be // needed when LLVM enables this `multivalue` feature by default. - if !cx.tcx.is_closure(instance.def_id()) { + if !cx.tcx.is_closure_or_coroutine(instance.def_id()) { let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); if abi == Abi::Wasm { function_features.push("+multivalue".to_string()); diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index cf47c94a81f..67bc86e4c90 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -99,7 +99,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { fn build(mut self: Box<Self>, output: &Path) -> bool { match self.build_with_llvm(output) { Ok(any_members) => any_members, - Err(e) => self.sess.emit_fatal(ArchiveBuildFailure { error: e }), + Err(e) => self.sess.dcx().emit_fatal(ArchiveBuildFailure { error: e }), } } } @@ -175,7 +175,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match std::fs::write(&def_file_path, def_file_content) { Ok(_) => {} Err(e) => { - sess.emit_fatal(ErrorWritingDEFFile { error: e }); + sess.dcx().emit_fatal(ErrorWritingDEFFile { error: e }); } }; @@ -217,14 +217,14 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { match dlltool_cmd.output() { Err(e) => { - sess.emit_fatal(ErrorCallingDllTool { + sess.dcx().emit_fatal(ErrorCallingDllTool { dlltool_path: dlltool.to_string_lossy(), error: e, }); } // dlltool returns '0' on failure, so check for error output instead. Ok(output) if !output.stderr.is_empty() => { - sess.emit_fatal(DlltoolFailImportLibrary { + sess.dcx().emit_fatal(DlltoolFailImportLibrary { dlltool_path: dlltool.to_string_lossy(), dlltool_args: dlltool_cmd .get_args() @@ -282,7 +282,7 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { }; if result == crate::llvm::LLVMRustResult::Failure { - sess.emit_fatal(ErrorCreatingImportLibrary { + sess.dcx().emit_fatal(ErrorCreatingImportLibrary { lib_name, error: llvm::last_error().unwrap_or("unknown LLVM error".to_string()), }); @@ -354,7 +354,7 @@ impl<'a> LlvmArchiveBuilder<'a> { let kind = kind .parse::<ArchiveKind>() .map_err(|_| kind) - .unwrap_or_else(|kind| self.sess.emit_fatal(UnknownArchiveKind { kind })); + .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); let mut additions = mem::take(&mut self.additions); let mut strings = Vec::new(); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index db297425b03..e9e8ade09b7 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{FatalError, Handler}; +use rustc_errors::{DiagCtxt, FatalError}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; @@ -47,7 +47,7 @@ pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { fn prepare_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> { let export_threshold = match cgcx.lto { // We're just doing LTO for our one crate @@ -60,7 +60,7 @@ fn prepare_lto( }; let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { - if info.level.is_below_threshold(export_threshold) || info.used { + if info.level.is_below_threshold(export_threshold) || info.used || info.used_compiler { Some(CString::new(name.as_str()).unwrap()) } else { None @@ -84,23 +84,23 @@ fn prepare_lto( // Make sure we actually can run LTO for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { - diag_handler.emit_err(LtoDisallowed); + dcx.emit_err(LtoDisallowed); return Err(FatalError); } else if *crate_type == CrateType::Dylib { if !cgcx.opts.unstable_opts.dylib_lto { - diag_handler.emit_err(LtoDylib); + dcx.emit_err(LtoDylib); return Err(FatalError); } } else if *crate_type == CrateType::ProcMacro { if !cgcx.opts.unstable_opts.dylib_lto { - diag_handler.emit_err(LtoProcMacro); + dcx.emit_err(LtoProcMacro); return Err(FatalError); } } } if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto { - diag_handler.emit_err(DynamicLinkingWithLTO); + dcx.emit_err(DynamicLinkingWithLTO); return Err(FatalError); } @@ -138,7 +138,7 @@ fn prepare_lto( upstream_modules.push((module, CString::new(name).unwrap())); } Err(e) => { - diag_handler.emit_err(e); + dcx.emit_err(e); return Err(FatalError); } } @@ -200,18 +200,11 @@ pub(crate) fn run_fat( modules: Vec<FatLtoInput<LlvmCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> { - let diag_handler = cgcx.create_diag_handler(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let dcx = cgcx.create_dcx(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?; let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); - fat_lto( - cgcx, - &diag_handler, - modules, - cached_modules, - upstream_modules, - &symbols_below_threshold, - ) + fat_lto(cgcx, &dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold) } /// Performs thin LTO by performing necessary global analysis and returning two @@ -222,8 +215,8 @@ pub(crate) fn run_thin( modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let dcx = cgcx.create_dcx(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?; let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>(); if cgcx.opts.cg.linker_plugin_lto.enabled() { @@ -232,14 +225,7 @@ pub(crate) fn run_thin( is deferred to the linker" ); } - thin_lto( - cgcx, - &diag_handler, - modules, - upstream_modules, - cached_modules, - &symbols_below_threshold, - ) + thin_lto(cgcx, &dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) { @@ -250,7 +236,7 @@ pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBu fn fat_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, modules: Vec<FatLtoInput<LlvmCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, @@ -316,7 +302,7 @@ fn fat_lto( let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); ModuleCodegen { - module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?, + module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?, name: name.into_string().unwrap(), kind: ModuleKind::Regular, } @@ -333,13 +319,8 @@ fn fat_lto( // The linking steps below may produce errors and diagnostics within LLVM // which we'd like to handle and print, so set up our diagnostic handlers // (which get unregistered when they go out of scope below). - let _handler = DiagnosticHandlers::new( - cgcx, - diag_handler, - llcx, - &module, - CodegenDiagnosticsStage::LTO, - ); + let _handler = + DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::LTO); // For all other modules we codegened we'll need to link them into our own // bitcode. All modules were codegened in their own LLVM context, however, @@ -367,9 +348,7 @@ fn fat_lto( }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker - .add(data) - .map_err(|()| write::llvm_err(diag_handler, LlvmError::LoadBitcode { name }))?; + linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; serialized_bitcode.push(bc_decoded); } drop(linker); @@ -452,7 +431,7 @@ impl Drop for Linker<'_> { /// they all go out of scope. fn thin_lto( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, modules: Vec<(String, ThinBuffer)>, serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, @@ -527,7 +506,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len() as u32, ) - .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::PrepareThinLtoContext))?; + .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; let data = ThinData(data); @@ -599,7 +578,7 @@ fn thin_lto( // session, overwriting the previous serialized data (if any). if let Some(path) = key_map_path { if let Err(err) = curr_key_map.save_to_file(&path) { - return Err(write::llvm_err(diag_handler, LlvmError::WriteThinLtoKey { err })); + return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); } } @@ -609,7 +588,7 @@ fn thin_lto( pub(crate) fn run_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: &mut ModuleCodegen<ModuleLlvm>, thin: bool, ) -> Result<(), FatalError> { @@ -631,13 +610,13 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddModuleFlag( module.module_llvm.llmod(), llvm::LLVMModFlagBehavior::Error, - "LTOPostLink\0".as_ptr().cast(), + c"LTOPostLink".as_ptr().cast(), 1, ); } let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage)?; + write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?; } debug!("lto done"); Ok(()) @@ -721,11 +700,11 @@ pub unsafe fn optimize_thin_module( thin_module: ThinModule<LlvmCodegenBackend>, cgcx: &CodegenContext<LlvmCodegenBackend>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { - let diag_handler = cgcx.create_diag_handler(); + let dcx = cgcx.create_dcx(); let module_name = &thin_module.shared.module_names[thin_module.idx]; let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); - let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, e))?; + let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&dcx, e))?; // Right now the implementation we've got only works over serialized // modules, so we create a fresh new LLVM context and parse the module @@ -733,7 +712,7 @@ pub unsafe fn optimize_thin_module( // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; + let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &dcx)? as *const _; let mut module = ModuleCodegen { module_llvm: ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) }, name: thin_module.name().to_string(), @@ -756,7 +735,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { - return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); + return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); } @@ -766,7 +745,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { - return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); + return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -776,7 +755,7 @@ pub unsafe fn optimize_thin_module( .prof .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { - return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); + return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -785,7 +764,7 @@ pub unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { - return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule)); + return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule)); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } @@ -797,7 +776,7 @@ pub unsafe fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - run_pass_manager(cgcx, &diag_handler, &mut module, true)?; + run_pass_manager(cgcx, &dcx, &mut module, true)?; save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } @@ -816,6 +795,9 @@ impl ThinLTOKeysMap { use std::io::Write; let file = File::create(path)?; let mut writer = io::BufWriter::new(file); + // The entries are loaded back into a hash map in `load_from_file()`, so + // the order in which we write them to file here does not matter. + #[allow(rustc::potential_query_instability)] for (module, key) in &self.keys { writeln!(writer, "{module} {key}")?; } @@ -865,10 +847,10 @@ pub fn parse_module<'a>( cx: &'a llvm::Context, name: &CStr, data: &[u8], - diag_handler: &Handler, + dcx: &DiagCtxt, ) -> Result<&'a llvm::Module, FatalError> { unsafe { llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) - .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::ParseBitcode)) + .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 36484c3c3fc..28a88dd2efe 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -39,7 +39,7 @@ impl OwnedTargetMachine { split_dwarf_file: &CStr, output_obj_file: &CStr, debug_info_compression: &CStr, - force_emulated_tls: bool, + use_emulated_tls: bool, args_cstr_buff: &[u8], ) -> Result<Self, LlvmError<'static>> { assert!(args_cstr_buff.len() > 0); @@ -71,7 +71,7 @@ impl OwnedTargetMachine { split_dwarf_file.as_ptr(), output_obj_file.as_ptr(), debug_info_compression.as_ptr(), - force_emulated_tls, + use_emulated_tls, args_cstr_buff.as_ptr() as *const c_char, args_cstr_buff.len(), ) diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 1a567c0fce8..45be85934b4 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -26,14 +26,14 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{FatalError, Handler, Level}; +use rustc_errors::{DiagCtxt, FatalError, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; -use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use crate::llvm::diagnostic::OptimizationDiagnosticKind; use libc::{c_char, c_int, c_uint, c_void, size_t}; @@ -45,15 +45,15 @@ use std::slice; use std::str; use std::sync::Arc; -pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError { +pub fn llvm_err<'a>(dcx: &rustc_errors::DiagCtxt, err: LlvmError<'a>) -> FatalError { match llvm::last_error() { - Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)), - None => handler.emit_almost_fatal(err), + Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)), + None => dcx.emit_almost_fatal(err), } } pub fn write_output_file<'ll>( - handler: &rustc_errors::Handler, + dcx: &rustc_errors::DiagCtxt, target: &'ll llvm::TargetMachine, pm: &llvm::PassManager<'ll>, m: &'ll llvm::Module, @@ -93,9 +93,7 @@ pub fn write_output_file<'ll>( } } - result - .into_result() - .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output })) + result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } } @@ -105,7 +103,7 @@ pub fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise()) + .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) } pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { @@ -128,7 +126,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMach tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise()) } pub fn to_llvm_opt_settings( @@ -223,7 +221,7 @@ pub fn target_machine_factory( let path_mapping = sess.source_map().path_mapping().clone(); - let force_emulated_tls = sess.target.force_emulated_tls; + let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); // copy the exe path, followed by path all into one buffer // null terminating them so we can use them as null terminated strings @@ -247,12 +245,12 @@ pub fn target_machine_factory( match sess.opts.debuginfo_compression { rustc_session::config::DebugInfoCompression::Zlib => { if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { - sess.emit_warning(UnknownCompression { algorithm: "zlib" }); + sess.dcx().emit_warning(UnknownCompression { algorithm: "zlib" }); } } rustc_session::config::DebugInfoCompression::Zstd => { if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { - sess.emit_warning(UnknownCompression { algorithm: "zstd" }); + sess.dcx().emit_warning(UnknownCompression { algorithm: "zstd" }); } } rustc_session::config::DebugInfoCompression::None => {} @@ -297,7 +295,7 @@ pub fn target_machine_factory( &split_dwarf_file, &output_obj_file, &debuginfo_compression, - force_emulated_tls, + use_emulated_tls, &args_cstr_buff, ) }) @@ -332,7 +330,7 @@ pub enum CodegenDiagnosticsStage { } pub struct DiagnosticHandlers<'a> { - data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a Handler), + data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a DiagCtxt), llcx: &'a llvm::Context, old_handler: Option<&'a llvm::DiagnosticHandler>, } @@ -340,7 +338,7 @@ pub struct DiagnosticHandlers<'a> { impl<'a> DiagnosticHandlers<'a> { pub fn new( cgcx: &'a CodegenContext<LlvmCodegenBackend>, - handler: &'a Handler, + dcx: &'a DiagCtxt, llcx: &'a llvm::Context, module: &ModuleCodegen<ModuleLlvm>, stage: CodegenDiagnosticsStage, @@ -375,7 +373,7 @@ impl<'a> DiagnosticHandlers<'a> { .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); let pgo_available = cgcx.opts.cg.profile_use.is_some(); - let data = Box::into_raw(Box::new((cgcx, handler))); + let data = Box::into_raw(Box::new((cgcx, dcx))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); llvm::LLVMRustContextConfigureDiagnosticHandler( @@ -418,7 +416,7 @@ fn report_inline_asm( cookie = 0; } let level = match level { - llvm::DiagnosticLevel::Error => Level::Error { lint: false }, + llvm::DiagnosticLevel::Error => Level::Error, llvm::DiagnosticLevel::Warning => Level::Warning(None), llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; @@ -429,7 +427,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void if user.is_null() { return; } - let (cgcx, diag_handler) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); + let (cgcx, dcx) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &DiagCtxt)); match llvm::diagnostic::Diagnostic::unpack(info) { llvm::diagnostic::InlineAsm(inline) => { @@ -437,7 +435,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void } llvm::diagnostic::Optimization(opt) => { - diag_handler.emit_note(FromLlvmOptimizationDiag { + dcx.emit_note(FromLlvmOptimizationDiag { filename: &opt.filename, line: opt.line, column: opt.column, @@ -459,14 +457,14 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.emit_warning(FromLlvmDiag { message }); + dcx.emit_warning(FromLlvmDiag { message }); } llvm::diagnostic::Unsupported(diagnostic_ref) => { let message = llvm::build_string(|s| { llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) }) .expect("non-UTF8 diagnostic"); - diag_handler.emit_err(FromLlvmDiag { message }); + dcx.emit_err(FromLlvmDiag { message }); } llvm::diagnostic::UnknownDiagnostic(..) => {} } @@ -507,7 +505,7 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> { pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: &ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, opt_level: config::OptLevel, @@ -588,13 +586,13 @@ pub(crate) unsafe fn llvm_optimize( llvm_plugins.as_ptr().cast(), llvm_plugins.len(), ); - result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses)) + result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. pub(crate) unsafe fn optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: &ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, ) -> Result<(), FatalError> { @@ -602,8 +600,7 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let _handlers = - DiagnosticHandlers::new(cgcx, diag_handler, llcx, module, CodegenDiagnosticsStage::Opt); + let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); @@ -621,14 +618,14 @@ pub(crate) unsafe fn optimize( _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, _ => llvm::OptStage::PreLinkNoLTO, }; - return llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage); + return llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage); } Ok(()) } pub(crate) fn link( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, mut modules: Vec<ModuleCodegen<ModuleLlvm>>, ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { use super::lto::{Linker, ModuleBuffer}; @@ -641,9 +638,9 @@ pub(crate) fn link( for module in elements { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); let buffer = ModuleBuffer::new(module.module_llvm.llmod()); - linker.add(buffer.data()).map_err(|()| { - llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name }) - })?; + linker + .add(buffer.data()) + .map_err(|()| llvm_err(dcx, LlvmError::SerializeModule { name: &module.name }))?; } drop(linker); Ok(modules.remove(0)) @@ -651,7 +648,7 @@ pub(crate) fn link( pub(crate) unsafe fn codegen( cgcx: &CodegenContext<LlvmCodegenBackend>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, ) -> Result<CompiledModule, FatalError> { @@ -662,13 +659,8 @@ pub(crate) unsafe fn codegen( let tm = &*module.module_llvm.tm; let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - let _handlers = DiagnosticHandlers::new( - cgcx, - diag_handler, - llcx, - &module, - CodegenDiagnosticsStage::Codegen, - ); + let _handlers = + DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen); if cgcx.msvc_imps_needed { create_msvc_imps(cgcx, llcx, llmod); @@ -726,7 +718,7 @@ pub(crate) unsafe fn codegen( .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); if let Err(err) = fs::write(&bc_out, data) { - diag_handler.emit_err(WriteBytecode { path: &bc_out, err }); + dcx.emit_err(WriteBytecode { path: &bc_out, err }); } } @@ -776,9 +768,7 @@ pub(crate) unsafe fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result - .into_result() - .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?; + result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?; } if config.emit_asm { @@ -797,7 +787,7 @@ pub(crate) unsafe fn codegen( }; with_codegen(tm, llmod, |cpm| { write_output_file( - diag_handler, + dcx, tm, cpm, llmod, @@ -832,7 +822,7 @@ pub(crate) unsafe fn codegen( with_codegen(tm, llmod, |cpm| { write_output_file( - diag_handler, + dcx, tm, cpm, llmod, @@ -847,12 +837,12 @@ pub(crate) unsafe fn codegen( EmitObj::Bitcode => { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); if let Err(err) = link_or_copy(&bc_out, &obj_out) { - diag_handler.emit_err(CopyBitcode { err }); + dcx.emit_err(CopyBitcode { err }); } if !config.emit_bc { debug!("removing_bitcode {:?}", bc_out); - ensure_removed(diag_handler, &bc_out); + ensure_removed(dcx, &bc_out); } } @@ -916,6 +906,7 @@ fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { cgcx.opts.target_triple.triple().contains("-aix") } +//FIXME use c string literals here too pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str { if target_is_apple(cgcx) { "__LLVM,__bitcode\0" @@ -994,7 +985,7 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.module\0".as_ptr().cast(), + c"rustc.embedded.module".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); @@ -1007,15 +998,15 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - "rustc.embedded.cmdline\0".as_ptr().cast(), + c"rustc.embedded.cmdline".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); let section = if is_apple { - "__LLVM,__cmdline\0" + c"__LLVM,__cmdline" } else if is_aix { - ".info\0" + c".info" } else { - ".llvmcmd\0" + c".llvmcmd" }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index b659fd02eec..5dc271ccddb 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -19,8 +19,6 @@ use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; -use cstr::cstr; - use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; @@ -110,11 +108,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // Create the llvm.used and llvm.compiler.used variables. if !cx.used_statics.borrow().is_empty() { - cx.create_used_variable_impl(cstr!("llvm.used"), &*cx.used_statics.borrow()); + cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow()); } if !cx.compiler_used_statics.borrow().is_empty() { cx.create_used_variable_impl( - cstr!("llvm.compiler.used"), + c"llvm.compiler.used", &*cx.compiler_used_statics.borrow(), ); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index acd85dd9a2d..7ed27b33dce 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -7,7 +7,6 @@ use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; use libc::{c_char, c_uint}; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -27,7 +26,6 @@ use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use std::borrow::Cow; -use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; @@ -47,13 +45,10 @@ impl Drop for Builder<'_, '_, '_> { } } -// FIXME(eddyb) use a checked constructor when they become `const fn`. -const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; - /// Empty string, to be used where LLVM expects an instruction name, indicating /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); +const UNNAMED: *const c_char = c"".as_ptr(); impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value; @@ -358,7 +353,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let new_kind = match ty.kind() { Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), - t @ (Uint(_) | Int(_)) => t.clone(), + t @ (Uint(_) | Int(_)) => *t, _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), }; @@ -563,10 +558,17 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Immediate(self.to_immediate(llval, place.layout)) } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { let b_offset = a.size(self).align_to(b.align(self).abi); - let pair_ty = place.layout.llvm_type(self); let mut load = |i, scalar: abi::Scalar, layout, align, offset| { - let llptr = self.struct_gep(pair_ty, place.llval, i as u64); + let llptr = if i == 0 { + place.llval + } else { + self.inbounds_gep( + self.type_i8(), + place.llval, + &[self.const_usize(b_offset.bytes())], + ) + }; let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); let load = self.load(llty, llptr, align); scalar_load_metadata(self, load, scalar, layout, offset); @@ -1012,14 +1014,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("cleanuppad"); let ret = unsafe { llvm::LLVMBuildCleanupPad( self.llbuilder, parent, args.as_ptr(), args.len() as c_uint, - name.as_ptr(), + c"cleanuppad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) @@ -1033,14 +1034,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { - let name = cstr!("catchpad"); let ret = unsafe { llvm::LLVMBuildCatchPad( self.llbuilder, parent, args.as_ptr(), args.len() as c_uint, - name.as_ptr(), + c"catchpad".as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for catchpad")) @@ -1052,14 +1052,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unwind: Option<&'ll BasicBlock>, handlers: &[&'ll BasicBlock], ) -> &'ll Value { - let name = cstr!("catchswitch"); let ret = unsafe { llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, handlers.len() as c_uint, - name.as_ptr(), + c"catchswitch".as_ptr(), ) }; let ret = ret.expect("LLVM does not have support for catchswitch"); @@ -1080,7 +1079,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { order: rustc_codegen_ssa::common::AtomicOrdering, failure_order: rustc_codegen_ssa::common::AtomicOrdering, weak: bool, - ) -> &'ll Value { + ) -> (&'ll Value, &'ll Value) { let weak = if weak { llvm::True } else { llvm::False }; unsafe { let value = llvm::LLVMBuildAtomicCmpXchg( @@ -1093,7 +1092,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::False, // SingleThreaded ); llvm::LLVMSetWeak(value, weak); - value + let val = self.extract_value(value, 0); + let success = self.extract_value(value, 1); + (val, success) } } fn atomic_rmw( diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index d1b643f4967..8173e41aff4 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -246,8 +246,8 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } Scalar::Ptr(ptr, _size) => { - let (alloc_id, offset) = ptr.into_parts(); - let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) { + let (prov, offset) = ptr.into_parts(); + let (base_addr, base_addr_space) = match self.tcx.global_alloc(prov.alloc_id()) { GlobalAlloc::Memory(alloc) => { let init = const_alloc_to_llvm(self, alloc); let alloc = alloc.inner(); diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index b6bc5395bf6..ec2fb2c6e54 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -8,7 +8,6 @@ use crate::llvm::{self, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -72,7 +71,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< } let mut next_offset = 0; - for &(offset, alloc_id) in alloc.provenance().ptrs().iter() { + for &(offset, prov) in alloc.provenance().ptrs().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; @@ -92,13 +91,10 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation< .expect("const_alloc_to_llvm: could not read relocation pointer") as u64; - let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer( - Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), - &cx.tcx, - ), + InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), Scalar::Initialized { value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size), @@ -135,10 +131,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Ok(min) => align = align.max(min), Err(err) => match err { AlignFromBytesError::NotPowerOfTwo(align) => { - cx.sess().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); + cx.sess().dcx().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align }); } AlignFromBytesError::TooLarge(align) => { - cx.sess().emit_err(InvalidMinimumAlignmentTooLarge { align }); + cx.sess().dcx().emit_err(InvalidMinimumAlignmentTooLarge { align }); } }, } @@ -173,7 +169,7 @@ fn check_and_apply_linkage<'ll, 'tcx>( let mut real_name = "_rust_extern_with_linkage_".to_string(); real_name.push_str(sym); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().emit_fatal(SymbolAlreadyDefined { + cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { span: cx.tcx.def_span(def_id), symbol_name: sym, }) @@ -476,9 +472,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { .all(|&byte| byte == 0); let sect_name = if all_bytes_are_zero { - cstr!("__DATA,__thread_bss") + c"__DATA,__thread_bss" } else { - cstr!("__DATA,__thread_data") + c"__DATA,__thread_data" }; llvm::LLVMSetSection(g, sect_name.as_ptr()); } @@ -507,7 +503,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, - "wasm.custom_sections\0".as_ptr().cast(), + c"wasm.custom_sections".as_ptr().cast(), val, ); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 92a8c00510b..1d1b6e6148d 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use crate::llvm_util; use crate::type_::Type; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; @@ -120,6 +119,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, + TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic, } } @@ -214,13 +214,13 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); + let avoid_plt = c"RtLibUseGOT".as_ptr().cast(); llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { - let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); + let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr().cast(); llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -231,7 +231,7 @@ pub unsafe fn create_module<'ll>( // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { - let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast(); + let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr().cast(); llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, @@ -242,7 +242,7 @@ pub unsafe fn create_module<'ll>( // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = "kcfi\0".as_ptr().cast(); + let kcfi = c"kcfi".as_ptr().cast(); llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); } @@ -255,7 +255,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 1, ) } @@ -264,7 +264,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "cfguard\0".as_ptr() as *const _, + c"cfguard".as_ptr() as *const _, 2, ) } @@ -282,26 +282,26 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, behavior, - "branch-target-enforcement\0".as_ptr().cast(), + c"branch-target-enforcement".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address\0".as_ptr().cast(), + c"sign-return-address".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address-all\0".as_ptr().cast(), + c"sign-return-address-all".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - "sign-return-address-with-bkey\0".as_ptr().cast(), + c"sign-return-address-with-bkey".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); } else { @@ -317,7 +317,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-branch\0".as_ptr().cast(), + c"cf-protection-branch".as_ptr().cast(), 1, ) } @@ -325,7 +325,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - "cf-protection-return\0".as_ptr().cast(), + c"cf-protection-return".as_ptr().cast(), 1, ) } @@ -334,7 +334,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Error, - "Virtual Function Elim\0".as_ptr().cast(), + c"Virtual Function Elim".as_ptr().cast(), 1, ); } @@ -344,7 +344,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - "ehcontguard\0".as_ptr() as *const _, + c"ehcontguard".as_ptr() as *const _, 1, ) } @@ -362,7 +362,7 @@ pub unsafe fn create_module<'ll>( ); llvm::LLVMAddNamedMetadataOperand( llmod, - cstr!("llvm.ident").as_ptr(), + c"llvm.ident".as_ptr(), llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1), ); @@ -510,14 +510,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { - let section = cstr!("llvm.metadata"); let array = self.const_array(self.type_ptr(), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, section.as_ptr()); + llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); } } } @@ -1015,9 +1014,9 @@ impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { - self.sess().emit_fatal(Spanned { span, node: err.into_diagnostic() }) + self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { - self.tcx.sess.emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) + self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) } } } @@ -1033,7 +1032,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.sess().emit_fatal(Spanned { span, node: err }) + self.tcx.dcx().emit_fatal(Spanned { span, node: err }) } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 7d69756181a..733a77d24c2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -10,7 +10,7 @@ use rustc_codegen_ssa::traits::{ BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, StaticMethods, }; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; @@ -30,7 +30,7 @@ const VAR_ALIGN_BYTES: usize = 8; pub struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: - RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, + RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -44,8 +44,8 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { pub fn take_function_coverage_map( &self, - ) -> FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { - self.function_coverage_map.replace(FxHashMap::default()) + ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { + self.function_coverage_map.replace(FxIndexMap::default()) } } @@ -85,6 +85,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let bx = self; + match coverage.kind { + // Marker statements have no effect during codegen, + // so return early and don't create `func_coverage`. + CoverageKind::SpanMarker => return, + // Match exhaustively to ensure that newly-added kinds are classified correctly. + CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } => {} + } + let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() else { @@ -100,6 +108,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let Coverage { kind } = coverage; match *kind { + CoverageKind::SpanMarker => unreachable!( + "unexpected marker statement {kind:?} should have caused an early return" + ), CoverageKind::CounterIncrement { id } => { func_coverage.mark_counter_id_seen(id); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 425e935bc9f..d82b1e1e721 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -30,14 +30,13 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, /// Allocates the global variable responsible for the .debug_gdb_scripts binary /// section. pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value { - let c_section_var_name = "__rustc_debug_gdb_scripts_section__\0"; - let section_var_name = &c_section_var_name[..c_section_var_name.len() - 1]; + let c_section_var_name = c"__rustc_debug_gdb_scripts_section__"; + let section_var_name = c_section_var_name.to_str().unwrap(); let section_var = unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; section_var.unwrap_or_else(|| { - let section_name = b".debug_gdb_scripts\0"; let mut section_contents = Vec::new(); // Add the pretty printers for the standard library first. @@ -70,7 +69,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); + llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast()); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index cf78fc56b49..76c9ac6614a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -17,10 +17,10 @@ use crate::debuginfo::utils::FatPtrKind; use crate::llvm; use crate::llvm::debuginfo::{ DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, + DebugNameTableKind, }; use crate::value::Value; -use cstr::cstr; use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; @@ -35,9 +35,10 @@ use rustc_middle::ty::{ use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::FileName; -use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_span::{FileNameDisplayPreference, SourceFile}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; +use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; use libc::{c_char, c_longlong, c_uint}; @@ -533,7 +534,7 @@ fn hex_encode(data: &[u8]) -> String { } pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { - let cache_key = Some((source_file.name_hash, source_file.src_hash)); + let cache_key = Some((source_file.stable_id, source_file.src_hash)); return debug_context(cx) .created_files .borrow_mut() @@ -606,7 +607,7 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> if let Ok(rel_path) = abs_path.strip_prefix(working_directory) { ( - working_directory.to_string_lossy().into(), + working_directory.to_string_lossy(), rel_path.to_string_lossy().into_owned(), ) } else { @@ -854,7 +855,6 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( use rustc_session::RemapFileNameExt; let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); let work_dir = tcx.sess.opts.working_dir.for_codegen(tcx.sess).to_string_lossy(); - let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames @@ -878,6 +878,17 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let split_name = split_name.to_str().unwrap(); let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); + let dwarf_version = + tcx.sess.opts.unstable_opts.dwarf_version.unwrap_or(tcx.sess.target.default_dwarf_version); + let is_dwarf_kind = + matches!(tcx.sess.target.debuginfo_kind, DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym); + // Don't emit `.debug_pubnames` and `.debug_pubtypes` on DWARFv4 or lower. + let debug_name_table_kind = if is_dwarf_kind && dwarf_version <= 4 { + DebugNameTableKind::None + } else { + DebugNameTableKind::Default + }; + unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( debug_context.builder, @@ -897,7 +908,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( producer.as_ptr().cast(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, - flags.as_ptr().cast(), + c"".as_ptr().cast(), 0, // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final @@ -907,6 +918,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( kind, 0, tcx.sess.opts.unstable_opts.split_dwarf_inlining, + debug_name_table_kind, ); if tcx.sess.opts.unstable_opts.profile { @@ -926,8 +938,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - let llvm_gcov_ident = cstr!("llvm.gcov"); - llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); } return unit_metadata; @@ -966,6 +977,27 @@ fn build_field_di_node<'ll, 'tcx>( } } +/// Returns the `DIFlags` corresponding to the visibility of the item identified by `did`. +/// +/// `DIFlags::Flag{Public,Protected,Private}` correspond to `DW_AT_accessibility` +/// (public/protected/private) aren't exactly right for Rust, but neither is `DW_AT_visibility` +/// (local/exported/qualified), and there's no way to set `DW_AT_visibility` in LLVM's API. +fn visibility_di_flags<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + did: DefId, + type_did: DefId, +) -> DIFlags { + let parent_did = cx.tcx.parent(type_did); + let visibility = cx.tcx.visibility(did); + match visibility { + Visibility::Public => DIFlags::FlagPublic, + // Private fields have a restricted visibility of the module containing the type. + Visibility::Restricted(did) if did == parent_did => DIFlags::FlagPrivate, + // `pub(crate)`/`pub(super)` visibilities are any other restricted visibility. + Visibility::Restricted(..) => DIFlags::FlagProtected, + } +} + /// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, @@ -989,7 +1021,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( &compute_debuginfo_type_name(cx.tcx, struct_type, false), size_and_align_of(struct_type_and_layout), Some(containing_scope), - DIFlags::FlagZero, + visibility_di_flags(cx, adt_def.did(), adt_def.did()), ), // Fields: |cx, owner| { @@ -1012,7 +1044,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( &field_name[..], (field_layout.size, field_layout.align.abi), struct_type_and_layout.fields.offset(i), - DIFlags::FlagZero, + visibility_di_flags(cx, f.did, adt_def.did()), type_di_node(cx, field_layout.ty), ) }) @@ -1034,7 +1066,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), _ => { bug!( diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 7ef185250a3..4792b0798df 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -26,8 +26,8 @@ use crate::{ enums::{tag_base_type, DiscrResult}, file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, UniqueTypeId}, - unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, - UNKNOWN_LINE_NUMBER, + unknown_file_metadata, visibility_di_flags, DINodeCreationResult, SmallVec, + NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER, }, utils::DIB, }, @@ -215,7 +215,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( &enum_type_name, cx.size_and_align_of(enum_type), NO_SCOPE_METADATA, - DIFlags::FlagZero, + visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()), ), |cx, enum_type_di_node| { match enum_type_and_layout.variants { @@ -320,6 +320,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_index: VariantIdx, ) -> SmallVec<&'ll DIType> { let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( cx, enum_type_and_layout, @@ -327,6 +328,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_index, enum_adt_def.variant(variant_index), variant_layout, + visibility_flags, ); let tag_base_type = cx.tcx.types.u32; @@ -364,7 +366,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( // since the later is sometimes smaller (if it has fewer fields). size_and_align_of(enum_type_and_layout), Size::ZERO, - DIFlags::FlagZero, + visibility_flags, variant_struct_type_wrapper_di_node, ), unsafe { @@ -376,7 +378,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, variant_names_type_di_node, - DIFlags::FlagZero, + visibility_flags, Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), tag_base_type_align.bits() as u32, ) @@ -403,6 +405,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( (variant_index, variant_name) }), ); + let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices .map(|variant_index| { @@ -417,6 +420,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( variant_index, variant_def, variant_layout, + visibility_flags, ); VariantFieldInfo { @@ -437,6 +441,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( tag_base_type, tag_field, untagged_variant_index, + visibility_flags, ) } @@ -674,7 +679,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( }; let (coroutine_def_id, coroutine_args) = match coroutine_type_and_layout.ty.kind() { - &ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine()), + &ty::Coroutine(def_id, args) => (def_id, args.as_coroutine()), _ => unreachable!(), }; @@ -744,6 +749,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( tag_base_type, tag_field, None, + DIFlags::FlagZero, ) } @@ -758,6 +764,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( tag_base_type: Ty<'tcx>, tag_field: usize, untagged_variant_index: Option<VariantIdx>, + di_flags: DIFlags, ) -> SmallVec<&'ll DIType> { let tag_base_type_di_node = type_di_node(cx, tag_base_type); let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); @@ -801,7 +808,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( align.bits() as u32, // Union fields are always at offset zero Size::ZERO.bits(), - DIFlags::FlagZero, + di_flags, variant_struct_type_wrapper, ) } @@ -835,7 +842,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( TAG_FIELD_NAME_128_LO, size_and_align, lo_offset, - DIFlags::FlagZero, + di_flags, type_di_node, )); @@ -855,7 +862,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( TAG_FIELD_NAME, cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), enum_type_and_layout.fields.offset(tag_field), - DIFlags::FlagZero, + di_flags, tag_base_type_di_node, )); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index df1df6d197e..7f671d1d061 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -250,6 +250,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( variant_index: VariantIdx, variant_def: &VariantDef, variant_layout: TyAndLayout<'tcx>, + di_flags: DIFlags, ) -> &'ll DIType { debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty); @@ -267,7 +268,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( // NOTE: We use size and align of enum_type, not from variant_layout: size_and_align_of(enum_type_and_layout), Some(enum_type_di_node), - DIFlags::FlagZero, + di_flags, ), |cx, struct_type_di_node| { (0..variant_layout.fields.count()) @@ -289,7 +290,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( &field_name, (field_layout.size, field_layout.align.abi), variant_layout.fields.offset(field_index), - DIFlags::FlagZero, + di_flags, type_di_node(cx, field_layout.ty), ) }) @@ -335,7 +336,7 @@ pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args, _) => args.as_coroutine(), + ty::Coroutine(_, args) => args.as_coroutine(), _ => unreachable!(), }; @@ -395,7 +396,7 @@ pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( }) .collect(); - state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() + state_specific_fields.into_iter().chain(common_fields).collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 130ca2727e4..3dbe820b8ff 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -7,8 +7,8 @@ use crate::{ enums::tag_base_type, file_metadata, size_and_align_of, type_di_node, type_map::{self, Stub, StubInfo, UniqueTypeId}, - unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, - UNKNOWN_LINE_NUMBER, + unknown_file_metadata, visibility_di_flags, DINodeCreationResult, SmallVec, + NO_GENERICS, UNKNOWN_LINE_NUMBER, }, utils::{create_DIArray, get_namespace_for_item, DIB}, }, @@ -63,6 +63,8 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type_and_layout = cx.layout_of(enum_type); let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); type_map::build_type_with_children( @@ -74,7 +76,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( &enum_type_name, size_and_align_of(enum_type_and_layout), Some(containing_scope), - DIFlags::FlagZero, + visibility_flags, ), |cx, enum_type_di_node| { // Build the struct type for each variant. These will be referenced by the @@ -92,6 +94,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( variant_index, enum_adt_def.variant(variant_index), enum_type_and_layout.for_variant(cx, variant_index), + visibility_flags, ), source_info: None, }) @@ -132,7 +135,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let coroutine_type = unique_type_id.expect_ty(); - let &ty::Coroutine(coroutine_def_id, _, _) = coroutine_type.kind() else { + let &ty::Coroutine(coroutine_def_id, _) = coroutine_type.kind() else { bug!("build_coroutine_di_node() called with non-coroutine type: `{:?}`", coroutine_type) }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 4832b147a54..d3a851b40c0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -1,6 +1,7 @@ #![doc = include_str!("doc.md")] use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; +use rustc_data_structures::unord::UnordMap; use self::metadata::{file_metadata, type_di_node}; use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; @@ -20,8 +21,6 @@ use crate::value::Value; use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::stable_hasher::Hash128; use rustc_data_structures::sync::Lrc; use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::IndexVec; @@ -32,7 +31,9 @@ use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; +use rustc_span::{ + BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, +}; use rustc_target::abi::Size; use libc::c_uint; @@ -61,7 +62,7 @@ pub struct CodegenUnitDebugContext<'ll, 'tcx> { llcontext: &'ll llvm::Context, llmod: &'ll llvm::Module, builder: &'ll mut DIBuilder<'ll>, - created_files: RefCell<FxHashMap<Option<(Hash128, SourceFileHash)>, &'ll DIFile>>, + created_files: RefCell<UnordMap<Option<(StableSourceFileId, SourceFileHash)>, &'ll DIFile>>, type_map: metadata::TypeMap<'ll, 'tcx>, namespace_map: RefCell<DefIdMap<&'ll DIScope>>, @@ -112,7 +113,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "Dwarf Version\0".as_ptr().cast(), + c"Dwarf Version".as_ptr().cast(), dwarf_version, ); } else { @@ -120,17 +121,16 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - "CodeView\0".as_ptr().cast(), + c"CodeView".as_ptr().cast(), 1, ) } // Prevent bitcode readers from deleting the debug info. - let ptr = "Debug Info Version\0".as_ptr(); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - ptr.cast(), + c"Debug Info Version".as_ptr().cast(), llvm::LLVMRustDebugMetadataVersion(), ); } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 164b12cf8d4..78c0725a637 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -84,7 +84,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_type: &'ll Type, ) -> &'ll Value { // Declare C ABI functions with the visibility used by C by default. - let visibility = if self.tcx.sess.target.default_hidden_visibility { + let visibility = if self.tcx.sess.default_hidden_visibility() { llvm::Visibility::Hidden } else { llvm::Visibility::Default @@ -107,7 +107,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unnamed: llvm::UnnamedAddr, fn_type: &'ll Type, ) -> &'ll Value { - let visibility = if self.tcx.sess.target.default_hidden_visibility { + let visibility = if self.tcx.sess.default_hidden_visibility() { llvm::Visibility::Hidden } else { llvm::Visibility::Default diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 10ca5ad802a..6e2c0dc21ad 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -4,9 +4,7 @@ use std::path::Path; use crate::fluent_generated as fluent; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{ - DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, Handler, IntoDiagnostic, -}; +use rustc_errors::{DiagCtxt, DiagnosticBuilder, EmissionGuarantee, IntoDiagnostic, Level}; use rustc_macros::{Diagnostic, Subdiagnostic}; use rustc_span::Span; @@ -101,15 +99,14 @@ pub(crate) struct DynamicLinkingWithLTO; pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>); -impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for ParseTargetMachineConfig<'_> { - fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { - let diag: DiagnosticBuilder<'_, EM> = self.0.into_diagnostic(sess); - let (message, _) = diag.styled_message().first().expect("`LlvmError` with no message"); - let message = sess.eagerly_translate_to_string(message.clone(), diag.args()); +impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for ParseTargetMachineConfig<'_> { + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { + let diag: DiagnosticBuilder<'_, G> = self.0.into_diagnostic(dcx, level); + let (message, _) = diag.messages().first().expect("`LlvmError` with no message"); + let message = dcx.eagerly_translate_to_string(message.clone(), diag.args()); - let mut diag = sess.struct_diagnostic(fluent::codegen_llvm_parse_target_machine_config); - diag.set_arg("error", message); - diag + DiagnosticBuilder::new(dcx, level, fluent::codegen_llvm_parse_target_machine_config) + .arg_mv("error", message) } } @@ -123,16 +120,20 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> { #[help(codegen_llvm_missing_features)] pub(crate) struct MissingFeatures; -impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> { - fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> { - let mut diag = sess.struct_err(fluent::codegen_llvm_target_feature_disable_or_enable); +impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for TargetFeatureDisableOrEnable<'_> { + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { + let mut diag = DiagnosticBuilder::new( + dcx, + level, + fluent::codegen_llvm_target_feature_disable_or_enable, + ); if let Some(span) = self.span { - diag.set_span(span); + diag.span(span); }; if let Some(missing_features) = self.missing_features { diag.subdiagnostic(missing_features); } - diag.set_arg("features", self.features.join(", ")); + diag.arg("features", self.features.join(", ")); diag } } @@ -183,8 +184,8 @@ pub enum LlvmError<'a> { pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String); -impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> { - fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> { +impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> { + fn into_diagnostic(self, dcx: &'_ DiagCtxt, level: Level) -> DiagnosticBuilder<'_, G> { use LlvmError::*; let msg_with_llvm_err = match &self.0 { WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err, @@ -201,10 +202,10 @@ impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> { PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err, ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err, }; - let mut diag = self.0.into_diagnostic(sess); - diag.set_primary_message(msg_with_llvm_err); - diag.set_arg("llvm_err", self.1); - diag + self.0 + .into_diagnostic(dcx, level) + .primary_message_mv(msg_with_llvm_err) + .arg_mv("llvm_err", self.1) } } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index cc7e78b9c62..a0f9d5cf7cd 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -10,7 +10,7 @@ use crate::value::Value; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; -use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; @@ -179,7 +179,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { unsafe { llvm::LLVMSetAlignment(load, align); } - self.to_immediate(load, self.layout_of(tp_ty)) + if !result.layout.is_zst() { + self.store(load, result.llval, result.align); + } + return; } sym::volatile_store => { let dst = args[0].deref(self.cx()); @@ -285,7 +288,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }, None => { - tcx.sess.emit_err(InvalidMonomorphization::BasicIntegerType { + tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty, @@ -921,7 +924,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ) -> Result<&'ll Value, ()> { macro_rules! return_error { ($diag: expr) => {{ - bx.sess().emit_err($diag); + bx.sess().dcx().emit_err($diag); return Err(()); }}; } @@ -946,6 +949,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); let arg_tys = sig.inputs(); + // Vectors must be immediates (non-power-of-2 #[repr(packed)] are not) + for (ty, arg) in arg_tys.iter().zip(args) { + if ty.is_simd() && !matches!(arg.val, OperandValue::Immediate(_)) { + return_error!(InvalidMonomorphization::SimdArgument { span, name, ty: *ty }); + } + } + if name == sym::simd_select_bitmask { let (len, _) = require_simd!(arg_tys[1], SimdArgument); @@ -1052,7 +1062,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .map(|(arg_idx, val)| { let idx = val.unwrap_leaf().try_to_i32().unwrap(); if idx >= i32::try_from(total_len).unwrap() { - bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + bx.sess().dcx().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { span, name, arg_idx: arg_idx as u64, @@ -1111,20 +1121,24 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let val = bx.const_get_elt(vector, i as u64); match bx.const_to_opt_u128(val, true) { None => { - bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexNotConstant { - span, - name, - arg_idx, - }); + bx.sess().dcx().emit_err( + InvalidMonomorphization::ShuffleIndexNotConstant { + span, + name, + arg_idx, + }, + ); None } Some(idx) if idx >= total_len => { - bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { - span, - name, - arg_idx, - total_len, - }); + bx.sess().dcx().emit_err( + InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, + arg_idx, + total_len, + }, + ); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1269,7 +1283,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ) -> Result<&'ll Value, ()> { macro_rules! return_error { ($diag: expr) => {{ - bx.sess().emit_err($diag); + bx.sess().dcx().emit_err($diag); return Err(()); }}; } @@ -1492,6 +1506,198 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(v); } + if name == sym::simd_masked_load { + // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T> + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Loads contiguous elements from memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument is a passthrough vector providing values for disabled lanes + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + require_simd!(ret_ty, SimdReturn); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The return type must match the last argument type + require!( + ret_ty == values_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty } + ); + + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Not, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, mask_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0"); + let fn_ty = bx + .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[1].immediate(), alignment, mask, args[2].immediate()], + None, + ); + return Ok(v); + } + + if name == sym::simd_masked_store { + // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> () + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + // Stores contiguous elements to memory behind `pointer`, but only for + // those lanes whose `mask` bit is enabled. + // The memory addresses corresponding to the “off” lanes are not accessed. + + // The element type of the "mask" argument must be a signed integer type of any width + let mask_ty = in_ty; + let (mask_len, mask_elem) = (in_len, in_elem); + + // The second argument must be a pointer matching the element type + let pointer_ty = arg_tys[1]; + + // The last argument specifies the values to store to memory + let values_ty = arg_tys[2]; + let (values_len, values_elem) = require_simd!(values_ty, SimdThird); + + // Of the same length: + require!( + values_len == mask_len, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len: mask_len, + in_ty: mask_ty, + arg_ty: values_ty, + out_len: values_len + } + ); + + // The second argument must be a mutable pointer type matching the element type + require!( + matches!( + pointer_ty.kind(), + ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut() + ), + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: values_elem, + second_arg: pointer_ty, + in_elem: values_elem, + in_ty: values_ty, + mutability: ExpectedPointerMutability::Mut, + } + ); + + require!( + matches!(mask_elem.kind(), ty::Int(_)), + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: values_elem, + third_arg: mask_ty, + } + ); + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[0].immediate(), i1xn), i1xn) + }; + + let ret_t = bx.type_void(); + + let llvm_pointer = bx.type_ptr(); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); + let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len); + + let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0"); + let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call( + fn_ty, + None, + None, + f, + &[args[2].immediate(), args[1].immediate(), alignment, mask], + None, + ); + return Ok(v); + } + if name == sym::simd_scatter { // simd_scatter(values: <N x T>, pointers: <N x *mut T>, // mask: <N x i{M}>) -> () diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 915cf31de08..a81056ed3ad 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -17,7 +17,6 @@ #![feature(never_type)] #![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] @@ -40,7 +39,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; +use rustc_errors::{DiagCtxt, ErrorGuaranteed, FatalError}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -200,10 +199,10 @@ impl WriteBackendMethods for LlvmCodegenBackend { } fn run_link( cgcx: &CodegenContext<Self>, - diag_handler: &Handler, + dcx: &DiagCtxt, modules: Vec<ModuleCodegen<Self::Module>>, ) -> Result<ModuleCodegen<Self::Module>, FatalError> { - back::write::link(cgcx, diag_handler, modules) + back::write::link(cgcx, dcx, modules) } fn run_fat_lto( cgcx: &CodegenContext<Self>, @@ -221,18 +220,18 @@ impl WriteBackendMethods for LlvmCodegenBackend { } unsafe fn optimize( cgcx: &CodegenContext<Self>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: &ModuleCodegen<Self::Module>, config: &ModuleConfig, ) -> Result<(), FatalError> { - back::write::optimize(cgcx, diag_handler, module, config) + back::write::optimize(cgcx, dcx, module, config) } fn optimize_fat( cgcx: &CodegenContext<Self>, module: &mut ModuleCodegen<Self::Module>, ) -> Result<(), FatalError> { - let diag_handler = cgcx.create_diag_handler(); - back::lto::run_pass_manager(cgcx, &diag_handler, module, false) + let dcx = cgcx.create_dcx(); + back::lto::run_pass_manager(cgcx, &dcx, module, false) } unsafe fn optimize_thin( cgcx: &CodegenContext<Self>, @@ -242,11 +241,11 @@ impl WriteBackendMethods for LlvmCodegenBackend { } unsafe fn codegen( cgcx: &CodegenContext<Self>, - diag_handler: &Handler, + dcx: &DiagCtxt, module: ModuleCodegen<Self::Module>, config: &ModuleConfig, ) -> Result<CompiledModule, FatalError> { - back::write::codegen(cgcx, diag_handler, module, config) + back::write::codegen(cgcx, dcx, module, config) } fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) { back::lto::prepare_thin(module) @@ -306,7 +305,9 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:"); - for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { + for name in + &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] + { writeln!(out, " {name}"); } writeln!(out); @@ -445,16 +446,16 @@ impl ModuleLlvm { cgcx: &CodegenContext<LlvmCodegenBackend>, name: &CStr, buffer: &[u8], - handler: &Handler, + dcx: &DiagCtxt, ) -> Result<Self, FatalError> { unsafe { let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?; + let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx)?; let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap()); let tm = match (cgcx.tm_factory)(tm_factory_config) { Ok(m) => m, Err(e) => { - return Err(handler.emit_almost_fatal(ParseTargetMachineConfig(e))); + return Err(dcx.emit_almost_fatal(ParseTargetMachineConfig(e))); } }; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6c3ccc9cf0d..aefca6b34f5 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -5,7 +5,7 @@ use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, - DebugEmissionKind, + DebugEmissionKind, DebugNameTableKind, }; use libc::{c_char, c_int, c_uint, size_t}; @@ -722,7 +722,7 @@ pub mod debuginfo { // These values **must** match with LLVMRustDIFlags!! bitflags! { #[repr(transparent)] - #[derive(Default)] + #[derive(Clone, Copy, Default)] pub struct DIFlags: u32 { const FlagZero = 0; const FlagPrivate = 1; @@ -751,7 +751,7 @@ pub mod debuginfo { // These values **must** match with LLVMRustDISPFlags!! bitflags! { #[repr(transparent)] - #[derive(Default)] + #[derive(Clone, Copy, Default)] pub struct DISPFlags: u32 { const SPFlagZero = 0; const SPFlagVirtual = 1; @@ -794,6 +794,15 @@ pub mod debuginfo { } } } + + /// LLVMRustDebugNameTableKind + #[derive(Clone, Copy)] + #[repr(C)] + pub enum DebugNameTableKind { + Default, + Gnu, + None, + } } use bitflags::bitflags; @@ -1812,6 +1821,7 @@ extern "C" { kind: DebugEmissionKind, DWOId: u64, SplitDebugInlining: bool, + DebugNameTableKind: DebugNameTableKind, ) -> &'a DIDescriptor; pub fn LLVMRustDIBuilderCreateFile<'a>( @@ -2159,7 +2169,7 @@ extern "C" { SplitDwarfFile: *const c_char, OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, - ForceEmulatedTls: bool, + UseEmulatedTls: bool, ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, ) -> *mut TargetMachine; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index eb69efb0d59..03b79a143cc 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -5,9 +5,6 @@ use crate::errors::{ }; use crate::llvm; use libc::c_int; -use rustc_codegen_ssa::target_features::{ - supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, -}; use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; @@ -17,6 +14,7 @@ use rustc_session::config::{PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use rustc_target::target_features::RUSTC_SPECIFIC_FEATURES; use std::ffi::{c_char, c_void, CStr, CString}; use std::path::Path; @@ -263,6 +261,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> { "sve2-bitperm", TargetFeatureFoldStrength::EnableOnly("neon"), ), + // The unaligned-scalar-mem feature was renamed to fast-unaligned-access. + ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => { + LLVMFeature::new("unaligned-scalar-mem") + } (_, s) => LLVMFeature::new(s), } } @@ -274,7 +276,7 @@ pub fn check_tied_features( features: &FxHashMap<&str, bool>, ) -> Option<&'static [&'static str]> { if !features.is_empty() { - for tied in tied_target_features(sess) { + for tied in sess.target.tied_target_features() { // Tied features must be set to the same value, or not set at all let mut tied_iter = tied.iter(); let enabled = features.get(tied_iter.next().unwrap()); @@ -290,7 +292,8 @@ pub fn check_tied_features( /// Must express features in the way Rust understands them pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { let target_machine = create_informational_target_machine(sess); - supported_target_features(sess) + sess.target + .supported_target_features() .iter() .filter_map(|&(feature, gate)| { if sess.is_nightly_build() || allow_unstable || gate.is_stable() { @@ -358,7 +361,9 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { fn print_target_features(out: &mut dyn PrintBackendInfo, sess: &Session, tm: &llvm::TargetMachine) { let mut llvm_target_features = llvm_target_features(tm); let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); - let mut rustc_target_features = supported_target_features(sess) + let mut rustc_target_features = sess + .target + .supported_target_features() .iter() .map(|(feature, _gate)| { // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. @@ -511,7 +516,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str ); // -Ctarget-features - let supported_features = supported_target_features(sess); + let supported_features = sess.target.supported_target_features(); let mut featsmap = FxHashMap::default(); let feats = sess .opts @@ -524,7 +529,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str Some(c @ ('+' | '-')) => c, Some(_) => { if diagnostics { - sess.emit_warning(UnknownCTargetFeaturePrefix { feature: s }); + sess.dcx().emit_warning(UnknownCTargetFeaturePrefix { feature: s }); } return None; } @@ -552,12 +557,12 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str } else { UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } }; - sess.emit_warning(unknown_feature); + sess.dcx().emit_warning(unknown_feature); } else if feature_state .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable()) { // An unstable feature. Warn about using it. - sess.emit_warning(UnstableCTargetFeature { feature }); + sess.dcx().emit_warning(UnstableCTargetFeature { feature }); } } @@ -593,7 +598,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str features.extend(feats); if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { - sess.emit_err(TargetFeatureDisableOrEnable { + sess.dcx().emit_err(TargetFeatureDisableOrEnable { features: f, span: None, missing_features: None, diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 01e82339664..f796ce0990f 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -26,6 +26,7 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { self.sess() + .dcx() .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 624ce6d8813..e88f4217c9d 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -26,16 +26,7 @@ fn uncached_llvm_type<'a, 'tcx>( let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } - Abi::ScalarPair(..) => { - return cx.type_struct( - &[ - layout.scalar_pair_element_llvm_type(cx, 0, false), - layout.scalar_pair_element_llvm_type(cx, 1, false), - ], - false, - ); - } - Abi::Uninhabited | Abi::Aggregate { .. } => {} + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalarPair(..) => {} } let name = match layout.ty.kind() { @@ -54,7 +45,7 @@ fn uncached_llvm_type<'a, 'tcx>( write!(&mut name, "::{}", def.variant(index).name).unwrap(); } } - if let (&ty::Coroutine(_, _, _), &Variants::Single { index }) = + if let (&ty::Coroutine(_, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap(); @@ -275,11 +266,25 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { - if let Abi::Scalar(scalar) = self.abi { - if scalar.is_bool() { - return cx.type_i1(); + match self.abi { + Abi::Scalar(scalar) => { + if scalar.is_bool() { + return cx.type_i1(); + } } - } + Abi::ScalarPair(..) => { + // An immediate pair always contains just the two elements, without any padding + // filler, as it should never be stored to memory. + return cx.type_struct( + &[ + self.scalar_pair_element_llvm_type(cx, 0, true), + self.scalar_pair_element_llvm_type(cx, 1, true), + ], + false, + ); + } + _ => {} + }; self.llvm_type(cx) } |