diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 39 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/archive.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 33 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 132 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/base.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/common.rs | 46 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 10 |
16 files changed, 207 insertions, 155 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 6e931aa0bc7..88daa025740 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -935,9 +935,10 @@ fn llvm_fixup_input<'ll, 'tcx>( } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16), + BackendRepr::SimdVector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count); let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); @@ -950,7 +951,7 @@ fn llvm_fixup_input<'ll, 'tcx>( } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - BackendRepr::Vector { .. }, + BackendRepr::SimdVector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), ( X86( @@ -985,7 +986,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - BackendRepr::Vector { element, count: count @ (8 | 16) }, + BackendRepr::SimdVector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } @@ -1022,7 +1023,7 @@ fn llvm_fixup_input<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - BackendRepr::Vector { element, count: count @ (4 | 8) }, + BackendRepr::SimdVector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_i16(), count)) } @@ -1095,9 +1096,10 @@ fn llvm_fixup_output<'ll, 'tcx>( } value } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16), + BackendRepr::SimdVector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count * 2); let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); @@ -1110,7 +1112,7 @@ fn llvm_fixup_output<'ll, 'tcx>( } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - BackendRepr::Vector { .. }, + BackendRepr::SimdVector { .. }, ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), ( X86( @@ -1141,7 +1143,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - BackendRepr::Vector { element, count: count @ (8 | 16) }, + BackendRepr::SimdVector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } @@ -1178,7 +1180,7 @@ fn llvm_fixup_output<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - BackendRepr::Vector { element, count: count @ (4 | 8) }, + BackendRepr::SimdVector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { bx.bitcast(value, bx.type_vector(bx.type_f16(), count)) } @@ -1239,9 +1241,10 @@ fn llvm_fixup_output_type<'ll, 'tcx>( let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16), + BackendRepr::SimdVector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(cx, element); cx.type_vector(elem_ty, count * 2) } @@ -1252,7 +1255,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( } ( X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), - BackendRepr::Vector { .. }, + BackendRepr::SimdVector { .. }, ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), ( X86( @@ -1280,7 +1283,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | X86InlineAsmRegClass::ymm_reg | X86InlineAsmRegClass::zmm_reg, ), - BackendRepr::Vector { element, count: count @ (8 | 16) }, + BackendRepr::SimdVector { element, count: count @ (8 | 16) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } @@ -1317,7 +1320,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>( | ArmInlineAsmRegClass::qreg_low4 | ArmInlineAsmRegClass::qreg_low8, ), - BackendRepr::Vector { element, count: count @ (4 | 8) }, + BackendRepr::SimdVector { element, count: count @ (4 | 8) }, ) if element.primitive() == Primitive::Float(Float::F16) => { cx.type_vector(cx.type_i16(), count) } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 93553f3f364..0a161442933 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -132,14 +132,33 @@ fn get_llvm_object_symbols( if err.is_null() { return Ok(true); } else { - return Err(unsafe { *Box::from_raw(err as *mut io::Error) }); + let error = unsafe { *Box::from_raw(err as *mut io::Error) }; + // These are the magic constants for LLVM bitcode files: + // https://github.com/llvm/llvm-project/blob/7eadc1960d199676f04add402bb0aa6f65b7b234/llvm/lib/BinaryFormat/Magic.cpp#L90-L97 + if buf.starts_with(&[0xDE, 0xCE, 0x17, 0x0B]) || buf.starts_with(&[b'B', b'C', 0xC0, 0xDE]) + { + // For LLVM bitcode, failure to read the symbols is not fatal. The bitcode may have been + // produced by a newer LLVM version that the one linked to rustc. This is fine provided + // that the linker does use said newer LLVM version. We skip writing the symbols for the + // bitcode to the symbol table of the archive. Traditional linkers don't like this, but + // newer linkers like lld, mold and wild ignore the symbol table anyway, so if they link + // against a new enough LLVM it will work out in the end. + // LLVM's archive writer also has this same behavior of only warning about invalid + // bitcode since https://github.com/llvm/llvm-project/pull/96848 + + // We don't have access to the DiagCtxt here to produce a nice warning in the correct format. + eprintln!("warning: Failed to read symbol table from LLVM bitcode: {}", error); + return Ok(true); + } else { + return Err(error); + } } unsafe extern "C" fn callback(state: *mut c_void, symbol_name: *const c_char) -> *mut c_void { let f = unsafe { &mut *(state as *mut &mut dyn FnMut(&[u8]) -> io::Result<()>) }; match f(unsafe { CStr::from_ptr(symbol_name) }.to_bytes()) { Ok(()) => std::ptr::null_mut(), - Err(err) => Box::into_raw(Box::new(err)) as *mut c_void, + Err(err) => Box::into_raw(Box::new(err) as Box<io::Error>) as *mut c_void, } } @@ -148,7 +167,7 @@ fn get_llvm_object_symbols( Box::into_raw(Box::new(io::Error::new( io::ErrorKind::Other, format!("LLVM error: {}", error.to_string_lossy()), - ))) as *mut c_void + )) as Box<io::Error>) as *mut c_void } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 99906ea7bce..668795191a2 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::path::Path; +use std::ptr::NonNull; use std::sync::Arc; use std::{io, iter, slice}; @@ -305,11 +306,8 @@ fn fat_lto( assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); - ModuleCodegen { - module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?, - name: name.into_string().unwrap(), - kind: ModuleKind::Regular, - } + let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?; + ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module) } }; { @@ -655,14 +653,14 @@ pub(crate) fn run_pass_manager( } unsafe { - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; } if cfg!(llvm_enzyme) && enable_ad { let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; unsafe { - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; } // This is the final IR, so people should be able to inspect the optimized autodiff output. @@ -729,6 +727,11 @@ impl ThinBuffer { ThinBuffer(buffer) } } + + pub unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer { + let mut ptr = NonNull::new(ptr).unwrap(); + ThinBuffer(unsafe { ptr.as_mut() }) + } } impl ThinBufferMethods for ThinBuffer { @@ -772,11 +775,11 @@ pub(crate) unsafe fn optimize_thin_module( // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?; - let mut module = ModuleCodegen { - module_llvm, - name: thin_module.name().to_string(), - kind: ModuleKind::Regular, - }; + let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); + // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. + if cgcx.config(ModuleKind::Regular).embed_bitcode() { + module.thin_lto_buffer = Some(thin_module.data().to_vec()); + } { let target = &*module.module_llvm.tm; let llmod = module.module_llvm.llmod(); @@ -793,7 +796,9 @@ pub(crate) unsafe fn optimize_thin_module( { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); - unsafe { llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) }; + unsafe { + llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target.raw()) + }; save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); } @@ -823,7 +828,7 @@ pub(crate) unsafe fn optimize_thin_module( let _timer = cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); if unsafe { - !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) + !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target.raw()) } { return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); } diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index f075f332462..dfde4595590 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,6 +1,5 @@ use std::ffi::{CStr, c_char}; use std::marker::PhantomData; -use std::ops::Deref; use std::ptr::NonNull; use rustc_data_structures::small_c_str::SmallCStr; @@ -80,12 +79,12 @@ impl OwnedTargetMachine { .map(|tm_unique| Self { tm_unique, phantom: PhantomData }) .ok_or_else(|| LlvmError::CreateTargetMachine { triple: SmallCStr::from(triple) }) } -} - -impl Deref for OwnedTargetMachine { - type Target = llvm::TargetMachine; - fn deref(&self) -> &Self::Target { + /// Returns inner `llvm::TargetMachine` type. + /// + /// This could be a `Deref` implementation, but `llvm::TargetMachine` is an extern type and + /// `Deref::Target: ?Sized`. + pub fn raw(&self) -> &llvm::TargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine. unsafe { self.tm_unique.as_ref() } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index b67890c0465..bead4c82a81 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,6 +1,7 @@ use std::ffi::{CStr, CString}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; +use std::ptr::null_mut; use std::sync::Arc; use std::{fs, slice, str}; @@ -15,7 +16,7 @@ use rustc_codegen_ssa::back::write::{ TargetMachineFactoryFn, }; use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; use rustc_errors::{DiagCtxtHandle, FatalError, Level}; @@ -551,6 +552,7 @@ pub(crate) unsafe fn llvm_optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'_>, module: &ModuleCodegen<ModuleLlvm>, + thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>, config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, @@ -584,7 +586,17 @@ pub(crate) unsafe fn llvm_optimize( vectorize_loop = config.vectorize_loop; } trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme); - let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); + if thin_lto_buffer.is_some() { + assert!( + matches!( + opt_stage, + llvm::OptStage::PreLinkNoLTO + | llvm::OptStage::PreLinkFatLTO + | llvm::OptStage::PreLinkThinLTO + ), + "the bitcode for LTO can only be obtained at the pre-link stage" + ); + } let pgo_gen_path = get_pgo_gen_path(config); let pgo_use_path = get_pgo_use_path(config); let pgo_sample_use_path = get_pgo_sample_use_path(config); @@ -637,14 +649,16 @@ pub(crate) unsafe fn llvm_optimize( let result = unsafe { llvm::LLVMRustOptimize( module.module_llvm.llmod(), - &*module.module_llvm.tm, + &*module.module_llvm.tm.raw(), to_pass_builder_opt_level(opt_level), opt_stage, cgcx.opts.cg.linker_plugin_lto.enabled(), config.no_prepopulate_passes, config.verify_llvm_ir, config.lint_llvm_ir, - using_thin_buffers, + thin_lto_buffer, + config.emit_thin_lto, + config.emit_thin_lto_summary, config.merge_functions, unroll_loops, vectorize_slp, @@ -675,7 +689,7 @@ pub(crate) unsafe fn llvm_optimize( pub(crate) unsafe fn optimize( cgcx: &CodegenContext<LlvmCodegenBackend>, dcx: DiagCtxtHandle<'_>, - module: &ModuleCodegen<ModuleLlvm>, + module: &mut ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, ) -> Result<(), FatalError> { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); @@ -705,9 +719,53 @@ pub(crate) unsafe fn optimize( // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; - return unsafe { - llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage) + // The embedded bitcode is used to run LTO/ThinLTO. + // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. + // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at + // this point. + let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular + && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)) + || config.emit_thin_lto_summary + { + Some(null_mut()) + } else { + None }; + unsafe { + llvm_optimize( + cgcx, + dcx, + module, + thin_lto_buffer.as_mut(), + config, + opt_level, + opt_stage, + autodiff_stage, + ) + }?; + if let Some(thin_lto_buffer) = thin_lto_buffer { + let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) }; + module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec()); + let bc_summary_out = + cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name); + if config.emit_thin_lto_summary + && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name() + { + let summary_data = thin_lto_buffer.thin_link_data(); + cgcx.prof.artifact_size( + "llvm_bitcode_summary", + thin_link_bitcode_filename.to_string_lossy(), + summary_data.len() as u64, + ); + let _timer = cgcx.prof.generic_activity_with_arg( + "LLVM_module_codegen_emit_bitcode_summary", + &*module.name, + ); + if let Err(err) = fs::write(&bc_summary_out, summary_data) { + dcx.emit_err(WriteBytecode { path: &bc_summary_out, err }); + } + } + } } Ok(()) } @@ -760,59 +818,41 @@ pub(crate) unsafe fn codegen( // otherwise requested. let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); - let bc_summary_out = - cgcx.output_filenames.temp_path(OutputType::ThinLinkBitcode, module_name); let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); if config.bitcode_needed() { - let _timer = cgcx - .prof - .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); - let thin = ThinBuffer::new(llmod, config.emit_thin_lto, config.emit_thin_lto_summary); - let data = thin.data(); - - if let Some(bitcode_filename) = bc_out.file_name() { - cgcx.prof.artifact_size( - "llvm_bitcode", - bitcode_filename.to_string_lossy(), - data.len() as u64, - ); - } - - if config.emit_thin_lto_summary - && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name() - { - let summary_data = thin.thin_link_data(); - cgcx.prof.artifact_size( - "llvm_bitcode_summary", - thin_link_bitcode_filename.to_string_lossy(), - summary_data.len() as u64, - ); - - let _timer = cgcx.prof.generic_activity_with_arg( - "LLVM_module_codegen_emit_bitcode_summary", - &*module.name, - ); - if let Err(err) = fs::write(&bc_summary_out, summary_data) { - dcx.emit_err(WriteBytecode { path: &bc_summary_out, err }); - } - } - if config.emit_bc || config.emit_obj == EmitObj::Bitcode { + let thin = { + let _timer = cgcx.prof.generic_activity_with_arg( + "LLVM_module_codegen_make_bitcode", + &*module.name, + ); + ThinBuffer::new(llmod, config.emit_thin_lto, false) + }; + let data = thin.data(); let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); + if let Some(bitcode_filename) = bc_out.file_name() { + cgcx.prof.artifact_size( + "llvm_bitcode", + bitcode_filename.to_string_lossy(), + data.len() as u64, + ); + } if let Err(err) = fs::write(&bc_out, data) { dcx.emit_err(WriteBytecode { path: &bc_out, err }); } } - if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) { + if config.embed_bitcode() && module.kind == ModuleKind::Regular { let _timer = cgcx .prof .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); + let thin_bc = + module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); unsafe { - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); + embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); } } } @@ -875,7 +915,7 @@ pub(crate) unsafe fn codegen( }; write_output_file( dcx, - tm, + tm.raw(), config.no_builtins, llmod, &path, @@ -909,7 +949,7 @@ pub(crate) unsafe fn codegen( write_output_file( dcx, - tm, + tm.raw(), config.no_builtins, llmod, &obj_out, diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index d35c7945bae..6bd27914dbd 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -13,10 +13,10 @@ use std::time::Instant; +use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; @@ -133,11 +133,7 @@ pub(crate) fn compile_codegen_unit( } } - ModuleCodegen { - name: cgu_name.to_string(), - module_llvm: llvm_module, - kind: ModuleKind::Regular, - } + ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module) } (module, cost) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7e8b2467989..55d34f5f2ef 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -360,7 +360,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { bundles.push(kcfi_bundle); } @@ -1388,7 +1388,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { bundles.push(kcfi_bundle); } @@ -1721,7 +1721,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); - if let Some(kcfi_bundle) = kcfi_bundle.as_deref() { + if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) { bundles.push(kcfi_bundle); } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 7af2815e64c..457e5452ce9 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -80,7 +80,7 @@ impl<'ll> Funclet<'ll> { } pub(crate) fn bundle(&self) -> &llvm::OperandBundle<'ll> { - &self.operand + self.operand.raw() } } @@ -130,10 +130,6 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMGetUndef(t) } } - fn is_undef(&self, v: &'ll Value) -> bool { - unsafe { llvm::LLVMIsUndef(v) == True } - } - fn const_poison(&self, t: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMGetPoison(t) } } @@ -212,28 +208,24 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { - let str_global = *self - .const_str_cache - .borrow_mut() - .raw_entry_mut() - .from_key(s) - .or_insert_with(|| { - let sc = self.const_bytes(s.as_bytes()); - let sym = self.generate_local_symbol_name("str"); - let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", sym); - }); - llvm::set_initializer(g, sc); - unsafe { - llvm::LLVMSetGlobalConstant(g, True); - llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); - } - llvm::set_linkage(g, llvm::Linkage::InternalLinkage); - // Cast to default address space if globals are in a different addrspace - let g = self.const_pointercast(g, self.type_ptr()); - (s.to_owned(), g) - }) - .1; + let mut const_str_cache = self.const_str_cache.borrow_mut(); + let str_global = const_str_cache.get(s).copied().unwrap_or_else(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + llvm::set_initializer(g, sc); + unsafe { + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); + } + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); + // Cast to default address space if globals are in a different addrspace + let g = self.const_pointercast(g, self.type_ptr()); + const_str_cache.insert(s.to_owned(), g); + g + }); let len = s.len(); (str_global, self.const_usize(len as u64)) } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ecef54d3165..9f8ec0ea526 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -219,7 +219,7 @@ pub(crate) unsafe fn create_module<'ll>( { let tm = crate::back::write::create_informational_target_machine(tcx.sess, false); unsafe { - llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); + llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm.raw()); } let llvm_data_layout = unsafe { llvm::LLVMGetDataLayoutStr(llmod) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 5e9f7f26f48..660fc7ec4c4 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -470,7 +470,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let layout = self.layout_of(tp_ty).layout; let use_integer_compare = match layout.backend_repr() { Scalar(_) | ScalarPair(_, _) => true, - Vector { .. } => false, + SimdVector { .. } => false, Memory { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), @@ -1329,7 +1329,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( )); } - if name == sym::simd_shuffle_generic { + if name == sym::simd_shuffle_const_generic { let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch(); let n = idx.len() as u64; @@ -1581,8 +1581,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), - sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), - sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), @@ -1615,8 +1613,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( | sym::simd_flog | sym::simd_floor | sym::simd_fma - | sym::simd_fpow - | sym::simd_fpowi | sym::simd_fsin | sym::simd_fsqrt | sym::simd_relaxed_fma diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index a15ec6000bc..e51d4852db2 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -12,7 +12,6 @@ #![feature(exact_size_is_empty)] #![feature(extern_types)] #![feature(file_buffered)] -#![feature(hash_raw_entry)] #![feature(if_let_guard)] #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] @@ -197,7 +196,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { unsafe fn optimize( cgcx: &CodegenContext<Self>, dcx: DiagCtxtHandle<'_>, - module: &ModuleCodegen<Self::Module>, + module: &mut ModuleCodegen<Self::Module>, config: &ModuleConfig, ) -> Result<(), FatalError> { unsafe { back::write::optimize(cgcx, dcx, module, config) } diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index ac8d4e57283..f6b23862907 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -43,10 +43,10 @@ pub use self::Enzyme_AD::*; #[cfg(llvm_enzyme)] pub mod Enzyme_AD { use libc::c_void; - extern "C" { + unsafe extern "C" { pub fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); } - extern "C" { + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; static mut EnzymePrintType: c_void; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6801f21aa77..39087a4d6f4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1056,7 +1056,6 @@ unsafe extern "C" { pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values - pub(crate) fn LLVMIsUndef(Val: &Value) -> Bool; pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type; pub(crate) fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); @@ -2435,7 +2434,9 @@ unsafe extern "C" { NoPrepopulatePasses: bool, VerifyIR: bool, LintIR: bool, - UseThinLTOBuffers: bool, + ThinLTOBuffer: Option<&mut *mut ThinLTOBuffer>, + EmitThinLTO: bool, + EmitThinLTOSummary: bool, MergeFunctions: bool, UnrollLoops: bool, SLPVectorize: bool, diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 5ec93424131..a36226b25a2 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -1,7 +1,6 @@ #![allow(non_snake_case)] use std::ffi::{CStr, CString}; -use std::ops::Deref; use std::ptr; use std::str::FromStr; use std::string::FromUtf8Error; @@ -355,6 +354,16 @@ impl<'a> OperandBundleOwned<'a> { }; OperandBundleOwned { raw: ptr::NonNull::new(raw).unwrap() } } + + /// Returns inner `OperandBundle` type. + /// + /// This could be a `Deref` implementation, but `OperandBundle` contains an extern type and + /// `Deref::Target: ?Sized`. + pub(crate) fn raw(&self) -> &OperandBundle<'a> { + // SAFETY: The returned reference is opaque and can only used for FFI. + // It is valid for as long as `&self` is. + unsafe { self.raw.as_ref() } + } } impl Drop for OperandBundleOwned<'_> { @@ -365,16 +374,6 @@ impl Drop for OperandBundleOwned<'_> { } } -impl<'a> Deref for OperandBundleOwned<'a> { - type Target = OperandBundle<'a>; - - fn deref(&self) -> &Self::Target { - // SAFETY: The returned reference is opaque and can only used for FFI. - // It is valid for as long as `&self` is. - unsafe { self.raw.as_ref() } - } -} - pub(crate) fn add_module_flag_u32( module: &Module, merge_behavior: ModuleFlagMergeBehavior, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index b4b5d6a5b19..5cc4f4ab9e6 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -281,6 +281,8 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("riscv32" | "riscv64", "zaamo") if get_version().0 < 19 => None, ("riscv32" | "riscv64", "zabha") if get_version().0 < 19 => None, ("riscv32" | "riscv64", "zalrsc") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zama16b") if get_version().0 < 19 => None, + ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None, // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => { Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512"))) @@ -329,7 +331,8 @@ pub(crate) fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<S if let Some(feat) = to_llvm_features(sess, feature) { for llvm_feature in feat { let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { + if !unsafe { llvm::LLVMRustHasFeature(target_machine.raw(), cstr.as_ptr()) } + { return false; } } @@ -451,8 +454,8 @@ pub(crate) fn print(req: &PrintRequest, out: &mut String, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess, false); match req.kind { - PrintKind::TargetCPUs => print_target_cpus(sess, &tm, out), - PrintKind::TargetFeatures => print_target_features(sess, &tm, out), + PrintKind::TargetCPUs => print_target_cpus(sess, tm.raw(), out), + PrintKind::TargetFeatures => print_target_features(sess, tm.raw(), out), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index ba01fbff385..4e7096da502 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -19,7 +19,7 @@ fn uncached_llvm_type<'a, 'tcx>( ) -> &'a Type { match layout.backend_repr { BackendRepr::Scalar(_) => bug!("handled elsewhere"), - BackendRepr::Vector { element, count } => { + BackendRepr::SimdVector { element, count } => { let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } @@ -171,7 +171,7 @@ pub(crate) trait LayoutLlvmExt<'tcx> { impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { match self.backend_repr { - BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true, + BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true, BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false, } } @@ -179,9 +179,9 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_scalar_pair(&self) -> bool { match self.backend_repr { BackendRepr::ScalarPair(..) => true, - BackendRepr::Scalar(_) | BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => { - false - } + BackendRepr::Scalar(_) + | BackendRepr::SimdVector { .. } + | BackendRepr::Memory { .. } => false, } } |