diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
18 files changed, 509 insertions, 616 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index e8076ce77ab..5f631405a9a 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -441,7 +441,9 @@ pub(crate) fn codegen_terminator_call<'tcx>( Err(instance) => Some(instance), } } - InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) => { + // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, + // it is `func returning noop future` + InstanceKind::DropGlue(_, None) => { // empty drop glue - a nop. let dest = target.expect("Non terminating drop_in_place_real???"); let ret_block = fx.get_block(dest); @@ -641,7 +643,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .flat_map(|arg_abi| arg_abi.get_abi_param(fx.tcx).into_iter()), ); - if fx.tcx.sess.target.is_like_osx && fx.tcx.sess.target.arch == "aarch64" { + if fx.tcx.sess.target.is_like_darwin && fx.tcx.sess.target.arch == "aarch64" { // Add any padding arguments needed for Apple AArch64. // There's no need to pad the argument list unless variadic arguments are actually being // passed. @@ -707,9 +709,8 @@ pub(crate) fn codegen_drop<'tcx>( let ty = drop_place.layout().ty; let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty); - if let ty::InstanceKind::DropGlue(_, None) | ty::InstanceKind::AsyncDropGlueCtorShim(_, None) = - drop_instance.def - { + // AsyncDropGlueCtorShim can't be here + if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def { // we don't actually need to drop anything } else { match ty.kind() { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 125a9201831..524e0d9fe35 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -8,8 +8,6 @@ use rustc_ast::InlineAsmOptions; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_index::IndexVec; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::InlineAsmMacro; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv}; @@ -18,7 +16,6 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; use crate::enable_verifier; -use crate::inline_asm::codegen_naked_asm; use crate::prelude::*; use crate::pretty_clif::CommentWriter; @@ -37,7 +34,7 @@ pub(crate) fn codegen_fn<'tcx>( cached_func: Function, module: &mut dyn Module, instance: Instance<'tcx>, -) -> Option<CodegenedFunction> { +) -> CodegenedFunction { debug_assert!(!instance.args.has_infer()); let symbol_name = tcx.symbol_name(instance).name.to_string(); @@ -54,38 +51,6 @@ pub(crate) fn codegen_fn<'tcx>( String::from_utf8_lossy(&buf).into_owned() }); - if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) { - assert_eq!(mir.basic_blocks.len(), 1); - assert!(mir.basic_blocks[START_BLOCK].statements.is_empty()); - - match &mir.basic_blocks[START_BLOCK].terminator().kind { - TerminatorKind::InlineAsm { - asm_macro: InlineAsmMacro::NakedAsm, - template, - operands, - options, - line_spans: _, - targets: _, - unwind: _, - } => { - codegen_naked_asm( - tcx, - cx, - module, - instance, - mir.basic_blocks[START_BLOCK].terminator().source_info.span, - &symbol_name, - template, - operands, - *options, - ); - } - _ => unreachable!(), - } - - return None; - } - // Declare function let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); @@ -166,7 +131,7 @@ pub(crate) fn codegen_fn<'tcx>( // Verify function verify_func(tcx, &clif_comments, &func); - Some(CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }) + CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx } } pub(crate) fn compile_fn( @@ -565,7 +530,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { | TerminatorKind::CoroutineDrop => { bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); } - TerminatorKind::Drop { place, target, unwind: _, replace: _ } => { + TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => { + assert!( + async_fut.is_none() && drop.is_none(), + "Async Drop must be expanded or reset to sync before codegen" + ); let drop_place = codegen_place(fx, *place); crate::abi::codegen_drop(fx, source_info, drop_place, *target); } @@ -729,8 +698,10 @@ fn codegen_stmt<'tcx>( let to_ty = fx.monomorphize(to_ty); fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { - ty.builtin_deref(true) - .is_some_and(|pointee_ty| has_ptr_meta(fx.tcx, pointee_ty)) + ty.builtin_deref(true).is_some_and(|pointee_ty| { + fx.tcx + .type_has_metadata(pointee_ty, ty::TypingEnv::fully_monomorphized()) + }) } if is_wide_ptr(fx, from_ty) { diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 766278d8718..abe2972ba0c 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -71,7 +71,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ }, ty::FnPtr(..) => pointer_ty(tcx), ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => { - if has_ptr_meta(tcx, *pointee_ty) { + if tcx.type_has_metadata(*pointee_ty, ty::TypingEnv::fully_monomorphized()) { return None; } else { pointer_ty(tcx) @@ -91,7 +91,7 @@ fn clif_pair_type_from_ty<'tcx>( (clif_type_from_ty(tcx, types[0])?, clif_type_from_ty(tcx, types[1])?) } ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => { - if has_ptr_meta(tcx, *pointee_ty) { + if tcx.type_has_metadata(*pointee_ty, ty::TypingEnv::fully_monomorphized()) { (pointer_ty(tcx), pointer_ty(tcx)) } else { return None; @@ -101,20 +101,6 @@ fn clif_pair_type_from_ty<'tcx>( }) } -/// Is a pointer to this type a wide ptr? -pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { - if ty.is_sized(tcx, ty::TypingEnv::fully_monomorphized()) { - return false; - } - - let tail = tcx.struct_tail_for_codegen(ty, ty::TypingEnv::fully_monomorphized()); - match tail.kind() { - ty::Foreign(..) => false, - ty::Str | ty::Slice(..) | ty::Dynamic(..) => true, - _ => bug!("unexpected unsized tail: {:?}", tail), - } -} - pub(crate) fn codegen_icmp_imm( fx: &mut FunctionCx<'_, '_, '_>, intcc: IntCC, diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs index d784f6e9d9e..d328b33a704 100644 --- a/compiler/rustc_codegen_cranelift/src/config.rs +++ b/compiler/rustc_codegen_cranelift/src/config.rs @@ -1,21 +1,10 @@ -/// The mode to use for compilation. -#[derive(Copy, Clone, Debug)] -pub enum CodegenMode { - /// AOT compile the crate. This is the default. - Aot, - /// JIT compile and execute the crate. - Jit, - /// JIT compile and execute the crate, but only compile functions the first time they are used. - JitLazy, -} - /// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. #[derive(Clone, Debug)] pub struct BackendConfig { /// Should the crate be AOT compiled or JIT executed. /// - /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. - pub codegen_mode: CodegenMode, + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=jit-mode`. + pub jit_mode: bool, /// When JIT mode is enable pass these arguments to the program. /// @@ -27,7 +16,7 @@ impl BackendConfig { /// Parse the configuration passed in using `-Cllvm-args`. pub fn from_opts(opts: &[String]) -> Result<Self, String> { let mut config = BackendConfig { - codegen_mode: CodegenMode::Aot, + jit_mode: false, jit_args: match std::env::var("CG_CLIF_JIT_ARGS") { Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(), Err(std::env::VarError::NotPresent) => vec![], @@ -43,20 +32,9 @@ impl BackendConfig { // testing cg_clif. continue; } - if let Some((name, value)) = opt.split_once('=') { - match name { - "mode" => { - config.codegen_mode = match value { - "aot" => CodegenMode::Aot, - "jit" => CodegenMode::Jit, - "jit-lazy" => CodegenMode::JitLazy, - _ => return Err(format!("Unknown codegen mode `{}`", value)), - }; - } - _ => return Err(format!("Unknown option `{}`", name)), - } - } else { - return Err(format!("Invalid option `{}`", opt)); + match &**opt { + "jit-mode" => config.jit_mode = true, + _ => return Err(format!("Unknown option `{}`", opt)), } } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index bcc70f4567f..c8527c3a57d 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -391,7 +391,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data.set_align(alloc.align.bytes()); if let Some(section_name) = section_name { - let (segment_name, section_name) = if tcx.sess.target.is_like_osx { + let (segment_name, section_name) = if tcx.sess.target.is_like_darwin { // See https://github.com/llvm/llvm-project/blob/main/llvm/lib/MC/MCSectionMachO.cpp let mut parts = section_name.as_str().split(','); let Some(segment_name) = parts.next() else { diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index bba6567774d..286e02b986b 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -58,7 +58,7 @@ impl DebugContext { // FIXME this should be configurable // macOS doesn't seem to support DWARF > 3 // 5 version is required for md5 file hash - version: if tcx.sess.target.is_like_osx { + version: if tcx.sess.target.is_like_darwin { 3 } else { // FIXME change to version 5 once the gdb and lldb shipping with the latest debian diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs index 017d7784dc0..25b922c8be4 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs @@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Ty, TyCtxt}; -use crate::{DebugContext, FullyMonomorphizedLayoutCx, has_ptr_meta}; +use crate::{DebugContext, FullyMonomorphizedLayoutCx}; #[derive(Default)] pub(crate) struct TypeDebugContext<'tcx> { @@ -129,7 +129,7 @@ impl DebugContext { let name = type_names::compute_debuginfo_type_name(tcx, ptr_type, true); - if !has_ptr_meta(tcx, ptr_type) { + if !tcx.type_has_metadata(ptr_type, ty::TypingEnv::fully_monomorphized()) { let pointer_type_id = self.dwarf.unit.add(self.dwarf.unit.root(), gimli::DW_TAG_pointer_type); let pointer_entry = self.dwarf.unit.get_mut(pointer_type_id); diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index fb7864ae612..5d07c94859f 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -22,7 +22,10 @@ use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; use rustc_metadata::EncodedMetadata; use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::mono::{ + CodegenUnit, Linkage as RLinkage, MonoItem, MonoItemData, Visibility, +}; use rustc_session::Session; use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType}; @@ -30,7 +33,7 @@ use crate::CodegenCx; use crate::base::CodegenedFunction; use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; use crate::debuginfo::TypeDebugContext; -use crate::global_asm::GlobalAsmConfig; +use crate::global_asm::{GlobalAsmConfig, GlobalAsmContext}; use crate::prelude::*; use crate::unwind_module::UnwindModule; @@ -169,8 +172,11 @@ fn produce_final_output_artifacts( if codegen_results.modules.len() == 1 { // 1) Only one codegen unit. In this case it's no difficulty // to copy `foo.0.x` to `foo.x`. - let module_name = Some(&codegen_results.modules[0].name[..]); - let path = crate_output.temp_path(output_type, module_name); + let path = crate_output.temp_path_for_cgu( + output_type, + &codegen_results.modules[0].name, + sess.invocation_temp.as_deref(), + ); let output = crate_output.path(output_type); if !output_type.is_text_output() && output.is_tty() { sess.dcx() @@ -183,22 +189,16 @@ fn produce_final_output_artifacts( ensure_removed(sess.dcx(), &path); } } else { - let extension = crate_output - .temp_path(output_type, None) - .extension() - .unwrap() - .to_str() - .unwrap() - .to_owned(); - if crate_output.outputs.contains_explicit_name(&output_type) { // 2) Multiple codegen units, with `--emit foo=some_name`. We have // no good solution for this case, so warn the user. - sess.dcx().emit_warn(ssa_errors::IgnoringEmitPath { extension }); + sess.dcx() + .emit_warn(ssa_errors::IgnoringEmitPath { extension: output_type.extension() }); } else if crate_output.single_output_file.is_some() { // 3) Multiple codegen units, with `-o some_name`. We have // no good solution for this case, so warn the user. - sess.dcx().emit_warn(ssa_errors::IgnoringOutput { extension }); + sess.dcx() + .emit_warn(ssa_errors::IgnoringOutput { extension: output_type.extension() }); } else { // 4) Multiple codegen units, but no explicit name. We // just leave the `foo.0.x` files in place. @@ -331,7 +331,7 @@ fn produce_final_output_artifacts( } fn make_module(sess: &Session, name: String) -> UnwindModule<ObjectModule> { - let isa = crate::build_isa(sess); + let isa = crate::build_isa(sess, false); let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); @@ -351,6 +351,7 @@ fn make_module(sess: &Session, name: String) -> UnwindModule<ObjectModule> { fn emit_cgu( output_filenames: &OutputFilenames, + invocation_temp: Option<&str>, prof: &SelfProfilerRef, name: String, module: UnwindModule<ObjectModule>, @@ -366,6 +367,7 @@ fn emit_cgu( let module_regular = emit_module( output_filenames, + invocation_temp, prof, product.object, ModuleKind::Regular, @@ -391,6 +393,7 @@ fn emit_cgu( fn emit_module( output_filenames: &OutputFilenames, + invocation_temp: Option<&str>, prof: &SelfProfilerRef, mut object: cranelift_object::object::write::Object<'_>, kind: ModuleKind, @@ -409,7 +412,7 @@ fn emit_module( object.set_section_data(comment_section, producer, 1); } - let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name)); + let tmp_file = output_filenames.temp_path_for_cgu(OutputType::Object, &name, invocation_temp); let file = match File::create(&tmp_file) { Ok(file) => file, Err(err) => return Err(format!("error creating object file: {}", err)), @@ -449,8 +452,11 @@ fn reuse_workproduct_for_cgu( cgu: &CodegenUnit<'_>, ) -> Result<ModuleCodegenResult, String> { let work_product = cgu.previous_work_product(tcx); - let obj_out_regular = - tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let obj_out_regular = tcx.output_filenames(()).temp_path_for_cgu( + OutputType::Object, + cgu.name().as_str(), + tcx.sess.invocation_temp.as_deref(), + ); let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( &tcx.sess, &work_product.saved_files.get("o").expect("no saved object file in work product"), @@ -527,19 +533,35 @@ fn codegen_cgu_content( let mut type_dbg = TypeDebugContext::default(); super::predefine_mono_items(tcx, module, &mono_items); let mut codegened_functions = vec![]; - for (mono_item, _) in mono_items { + for (mono_item, item_data) in mono_items { match mono_item { - MonoItem::Fn(inst) => { - if let Some(codegened_function) = crate::base::codegen_fn( + MonoItem::Fn(instance) => { + if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) + { + rustc_codegen_ssa::mir::naked_asm::codegen_naked_asm( + &mut GlobalAsmContext { tcx, global_asm: &mut cx.global_asm }, + instance, + MonoItemData { + linkage: RLinkage::External, + visibility: if item_data.linkage == RLinkage::Internal { + Visibility::Hidden + } else { + item_data.visibility + }, + ..item_data + }, + ); + continue; + } + let codegened_function = crate::base::codegen_fn( tcx, &mut cx, &mut type_dbg, Function::new(), module, - inst, - ) { - codegened_functions.push(codegened_function); - } + instance, + ); + codegened_functions.push(codegened_function); } MonoItem::Static(def_id) => { let data_id = crate::constant::codegen_static(tcx, module, def_id); @@ -548,7 +570,10 @@ fn codegen_cgu_content( } } MonoItem::GlobalAsm(item_id) => { - crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id); + rustc_codegen_ssa::base::codegen_global_asm( + &mut GlobalAsmContext { tcx, global_asm: &mut cx.global_asm }, + item_id, + ); } } } @@ -595,13 +620,19 @@ fn module_codegen( let global_asm_object_file = profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { - crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm) + crate::global_asm::compile_global_asm( + &global_asm_config, + &cgu_name, + &cx.global_asm, + cx.invocation_temp.as_deref(), + ) })?; let codegen_result = profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { emit_cgu( &global_asm_config.output_filenames, + cx.invocation_temp.as_deref(), &profiler, cgu_name, module, @@ -626,8 +657,11 @@ fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> Compiled .as_str() .to_string(); - let tmp_file = - tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + let tmp_file = tcx.output_filenames(()).temp_path_for_cgu( + OutputType::Metadata, + &metadata_cgu_name, + tcx.sess.invocation_temp.as_deref(), + ); let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx); let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name); @@ -657,6 +691,7 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> { match emit_module( tcx.output_filenames(()), + tcx.sess.invocation_temp.as_deref(), &tcx.sess.prof, product.object, ModuleKind::Allocator, @@ -728,26 +763,27 @@ pub(crate) fn run_aot( let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(todo_cgus.len())); - let modules = tcx.sess.time("codegen mono items", || { - let mut modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| { - let dep_node = cgu.codegen_dep_node(tcx); - tcx.dep_graph - .with_task( + let modules: Vec<_> = + tcx.sess.time("codegen mono items", || { + let modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| { + let dep_node = cgu.codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( dep_node, tcx, (global_asm_config.clone(), cgu.name(), concurrency_limiter.acquire(tcx.dcx())), module_codegen, Some(rustc_middle::dep_graph::hash_result), - ) - .0 - }); - modules.extend( - done_cgus + ); + IntoDynSyncSend(module) + }); + modules .into_iter() - .map(|(_, cgu)| OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))), - ); - modules - }); + .map(|module| module.0) + .chain(done_cgus.into_iter().map(|(_, cgu)| { + OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu)) + })) + .collect() + }); let allocator_module = emit_allocator_module(tcx); diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 57c88f4b0f9..e368cf4386d 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -1,75 +1,27 @@ //! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. -use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; -use std::sync::{Mutex, OnceLock, mpsc}; -use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_jit::{JITBuilder, JITModule}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; use rustc_session::Session; use rustc_span::sym; +use crate::CodegenCx; use crate::debuginfo::TypeDebugContext; use crate::prelude::*; use crate::unwind_module::UnwindModule; -use crate::{CodegenCx, CodegenMode}; -struct JitState { - jit_module: UnwindModule<JITModule>, -} - -thread_local! { - static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) }; -} - -/// The Sender owned by the rustc thread -static GLOBAL_MESSAGE_SENDER: OnceLock<Mutex<mpsc::Sender<UnsafeMessage>>> = OnceLock::new(); - -/// A message that is sent from the jitted runtime to the rustc thread. -/// Senders are responsible for upholding `Send` semantics. -enum UnsafeMessage { - /// Request that the specified `Instance` be lazily jitted. - /// - /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after - /// this message is sent. - JitFn { - instance_ptr: *const Instance<'static>, - trampoline_ptr: *const u8, - tx: mpsc::Sender<*const u8>, - }, -} -unsafe impl Send for UnsafeMessage {} - -impl UnsafeMessage { - /// Send the message. - fn send(self) { - thread_local! { - /// The Sender owned by the local thread - static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> = - GLOBAL_MESSAGE_SENDER - .get().unwrap() - .lock().unwrap() - .clone(); - } - LOCAL_MESSAGE_SENDER.with(|sender| { - sender.send(self).expect("rustc thread hung up before lazy JIT request was sent") - }) - } -} - -fn create_jit_module(tcx: TyCtxt<'_>, hotswap: bool) -> (UnwindModule<JITModule>, CodegenCx) { +fn create_jit_module(tcx: TyCtxt<'_>) -> (UnwindModule<JITModule>, CodegenCx) { let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); - let isa = crate::build_isa(tcx.sess); + let isa = crate::build_isa(tcx.sess, true); let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); - jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info)); - jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8); let mut jit_module = UnwindModule::new(JITModule::new(jit_builder), false); let cx = crate::CodegenCx::new(tcx, jit_module.isa(), false, sym::dummy_cgu_name); @@ -79,7 +31,7 @@ fn create_jit_module(tcx: TyCtxt<'_>, hotswap: bool) -> (UnwindModule<JITModule> (jit_module, cx) } -pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec<String>) -> ! { +pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.dcx().fatal("JIT mode doesn't work with `cargo check`"); } @@ -88,8 +40,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec< tcx.dcx().fatal("can't jit non-executable crate"); } - let (mut jit_module, mut cx) = - create_jit_module(tcx, matches!(codegen_mode, CodegenMode::JitLazy)); + let (mut jit_module, mut cx) = create_jit_module(tcx); let mut cached_context = Context::new(); let cgus = tcx.collect_and_partition_mono_items(()).codegen_units; @@ -105,21 +56,15 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec< super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { - MonoItem::Fn(inst) => match codegen_mode { - CodegenMode::Aot => unreachable!(), - CodegenMode::Jit => { - codegen_and_compile_fn( - tcx, - &mut cx, - &mut cached_context, - &mut jit_module, - inst, - ); - } - CodegenMode::JitLazy => { - codegen_shim(tcx, &mut cached_context, &mut jit_module, inst) - } - }, + MonoItem::Fn(inst) => { + codegen_and_compile_fn( + tcx, + &mut cx, + &mut cached_context, + &mut jit_module, + inst, + ); + } MonoItem::Static(def_id) => { crate::constant::codegen_static(tcx, &mut jit_module, def_id); } @@ -161,41 +106,17 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec< let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id); - LAZY_JIT_STATE.with(|lazy_jit_state| { - let mut lazy_jit_state = lazy_jit_state.borrow_mut(); - assert!(lazy_jit_state.is_none()); - *lazy_jit_state = Some(JitState { jit_module }); - }); - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_start) }; - let (tx, rx) = mpsc::channel(); - GLOBAL_MESSAGE_SENDER.set(Mutex::new(tx)).unwrap(); + let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); - // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages - // (eg to lazily JIT further functions as required) - std::thread::spawn(move || { - let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); + // Push a null pointer as a terminating argument. This is required by POSIX and + // useful as some dynamic linkers use it as a marker to jump over. + argv.push(std::ptr::null()); - // Push a null pointer as a terminating argument. This is required by POSIX and - // useful as some dynamic linkers use it as a marker to jump over. - argv.push(std::ptr::null()); - - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - }); - - // Handle messages - loop { - match rx.recv().unwrap() { - // lazy JIT compilation request - compile requested instance and return pointer to result - UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } => { - tx.send(jit_fn(instance_ptr, trampoline_ptr)) - .expect("jitted runtime hung up before response to lazy JIT request was sent"); - } - } - } + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } pub(crate) fn codegen_and_compile_fn<'tcx>( @@ -205,6 +126,11 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( module: &mut dyn Module, instance: Instance<'tcx>, ) { + if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) { + tcx.dcx() + .span_fatal(tcx.def_span(instance.def_id()), "Naked asm is not supported in JIT mode"); + } + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( tcx.prof.clone(), ))); @@ -214,71 +140,18 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); - if let Some(codegened_func) = crate::base::codegen_fn( + let codegened_func = crate::base::codegen_fn( tcx, cx, &mut TypeDebugContext::default(), cached_func, module, instance, - ) { - crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func); - } + ); + crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func); }); } -extern "C" fn clif_jit_fn( - instance_ptr: *const Instance<'static>, - trampoline_ptr: *const u8, -) -> *const u8 { - // send the JIT request to the rustc thread, with a channel for the response - let (tx, rx) = mpsc::channel(); - UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx }.send(); - - // block on JIT compilation result - rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request") -} - -fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> *const u8 { - rustc_middle::ty::tls::with(|tcx| { - // lift is used to ensure the correct lifetime for instance. - let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - - LAZY_JIT_STATE.with(|lazy_jit_state| { - let mut lazy_jit_state = lazy_jit_state.borrow_mut(); - let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); - let jit_module = &mut lazy_jit_state.jit_module; - - let name = tcx.symbol_name(instance).name; - let sig = crate::abi::get_function_sig( - tcx, - jit_module.target_config().default_call_conv, - instance, - ); - let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); - - let current_ptr = jit_module.module.read_got_entry(func_id); - - // If the function's GOT entry has already been updated to point at something other - // than the shim trampoline, don't re-jit but just return the new pointer instead. - // This does not need synchronization as this code is executed only by a sole rustc - // thread. - if current_ptr != trampoline_ptr { - return current_ptr; - } - - jit_module.module.prepare_for_function_redefine(func_id).unwrap(); - - let mut cx = crate::CodegenCx::new(tcx, jit_module.isa(), false, sym::dummy_cgu_name); - codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance); - - assert!(cx.global_asm.is_empty()); - jit_module.finalize_definitions(); - jit_module.module.get_finalized_function(func_id) - }) - }) -} - fn dep_symbol_lookup_fn( sess: &Session, crate_info: CrateInfo, @@ -326,57 +199,3 @@ fn dep_symbol_lookup_fn( None }) } - -fn codegen_shim<'tcx>( - tcx: TyCtxt<'tcx>, - cached_context: &mut Context, - module: &mut UnwindModule<JITModule>, - inst: Instance<'tcx>, -) { - let pointer_type = module.target_config().pointer_type(); - - let name = tcx.symbol_name(inst).name; - let sig = crate::abi::get_function_sig(tcx, module.target_config().default_call_conv, inst); - let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); - - let instance_ptr = Box::into_raw(Box::new(inst)); - - let jit_fn = module - .declare_function( - "__clif_jit_fn", - Linkage::Import, - &Signature { - call_conv: module.target_config().default_call_conv, - params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)], - returns: vec![AbiParam::new(pointer_type)], - }, - ) - .unwrap(); - - let context = cached_context; - context.clear(); - let trampoline = &mut context.func; - trampoline.signature = sig.clone(); - - let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - - let trampoline_fn = module.declare_func_in_func(func_id, trampoline_builder.func); - let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); - let sig_ref = trampoline_builder.func.import_signature(sig); - - let entry_block = trampoline_builder.create_block(); - trampoline_builder.append_block_params_for_function_params(entry_block); - let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec(); - - trampoline_builder.switch_to_block(entry_block); - let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64); - let trampoline_ptr = trampoline_builder.ins().func_addr(pointer_type, trampoline_fn); - let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr, trampoline_ptr]); - let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; - let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args); - let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); - trampoline_builder.ins().return_(&ret_vals); - - module.define_function(func_id, context).unwrap(); -} diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index 9ea92c300f8..203b443269f 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -7,102 +7,139 @@ use std::process::{Command, Stdio}; use std::sync::Arc; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; -use rustc_hir::{InlineAsmOperand, ItemId}; -use rustc_middle::mir::interpret::ErrorHandled; +use rustc_codegen_ssa::traits::{AsmCodegenMethods, GlobalAsmOperandRef}; +use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers, +}; use rustc_session::config::{OutputFilenames, OutputType}; use rustc_target::asm::InlineAsmArch; use crate::prelude::*; -pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) { - let item = tcx.hir_item(item_id); - if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind { - let is_x86 = - matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64); - - if is_x86 { - if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { - global_asm.push_str("\n.intel_syntax noprefix\n"); - } else { - global_asm.push_str("\n.att_syntax\n"); - } +pub(crate) struct GlobalAsmContext<'a, 'tcx> { + pub tcx: TyCtxt<'tcx>, + pub global_asm: &'a mut String, +} + +impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> { + fn codegen_global_asm( + &mut self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef<'tcx>], + options: InlineAsmOptions, + _line_spans: &[Span], + ) { + codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options); + } + + fn mangled_name(&self, instance: Instance<'tcx>) -> String { + let symbol_name = self.tcx.symbol_name(instance).name.to_owned(); + if self.tcx.sess.target.is_like_darwin { format!("_{symbol_name}") } else { symbol_name } + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> { + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + self.tcx.sess.dcx().span_fatal(span, err.to_string()) + } else { + self.tcx + .sess + .dcx() + .span_fatal(span, format!("failed to get layout for `{}`: {}", ty, err)) } - for piece in asm.template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), - InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: op_sp } => { - match asm.operands[operand_idx].0 { - InlineAsmOperand::Const { ref anon_const } => { - match tcx.const_eval_poly(anon_const.def_id.to_def_id()) { - Ok(const_value) => { - let ty = tcx - .typeck_body(anon_const.body) - .node_type(anon_const.hir_id); - let string = rustc_codegen_ssa::common::asm_const_to_str( - tcx, - op_sp, - const_value, - FullyMonomorphizedLayoutCx(tcx).layout_of(ty), - ); - global_asm.push_str(&string); - } - Err(ErrorHandled::Reported { .. }) => { - // An error has already been reported and compilation is - // guaranteed to fail if execution hits this path. - } - Err(ErrorHandled::TooGeneric(_)) => { - span_bug!(op_sp, "asm const cannot be resolved; too generic"); - } - } - } - InlineAsmOperand::SymFn { expr } => { - if cfg!(not(feature = "inline_asm_sym")) { - tcx.dcx().span_err( - item.span, - "asm! and global_asm! sym operands are not yet supported", - ); - } - - let ty = tcx.typeck(item_id.owner_id).expr_ty(expr); - let instance = match ty.kind() { - &ty::FnDef(def_id, args) => Instance::new(def_id, args), - _ => span_bug!(op_sp, "asm sym is not a function"), - }; - let symbol = tcx.symbol_name(instance); - // FIXME handle the case where the function was made private to the - // current codegen unit - global_asm.push_str(symbol.name); - } - InlineAsmOperand::SymStatic { path: _, def_id } => { - if cfg!(not(feature = "inline_asm_sym")) { - tcx.dcx().span_err( - item.span, - "asm! and global_asm! sym operands are not yet supported", - ); - } - - let instance = Instance::mono(tcx, def_id); - let symbol = tcx.symbol_name(instance); - global_asm.push_str(symbol.name); + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> { + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + FullyMonomorphizedLayoutCx(self.tcx).handle_fn_abi_err(err, span, fn_abi_request) + } +} + +impl<'tcx> HasTyCtxt<'tcx> for GlobalAsmContext<'_, 'tcx> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx> rustc_abi::HasDataLayout for GlobalAsmContext<'_, 'tcx> { + fn data_layout(&self) -> &rustc_abi::TargetDataLayout { + &self.tcx.data_layout + } +} + +impl<'tcx> HasTypingEnv<'tcx> for GlobalAsmContext<'_, 'tcx> { + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + ty::TypingEnv::fully_monomorphized() + } +} + +fn codegen_global_asm_inner<'tcx>( + tcx: TyCtxt<'tcx>, + global_asm: &mut String, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef<'tcx>], + options: InlineAsmOptions, +) { + let is_x86 = matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64); + + if is_x86 { + if !options.contains(InlineAsmOptions::ATT_SYNTAX) { + global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + global_asm.push_str("\n.att_syntax\n"); + } + } + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => { + match operands[operand_idx] { + GlobalAsmOperandRef::Const { ref string } => { + global_asm.push_str(string); + } + GlobalAsmOperandRef::SymFn { instance } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx().span_err( + span, + "asm! and global_asm! sym operands are not yet supported", + ); } - InlineAsmOperand::In { .. } - | InlineAsmOperand::Out { .. } - | InlineAsmOperand::InOut { .. } - | InlineAsmOperand::SplitInOut { .. } - | InlineAsmOperand::Label { .. } => { - span_bug!(op_sp, "invalid operand type for global_asm!") + + let symbol = tcx.symbol_name(instance); + // FIXME handle the case where the function was made private to the + // current codegen unit + global_asm.push_str(symbol.name); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + if cfg!(not(feature = "inline_asm_sym")) { + tcx.dcx().span_err( + span, + "asm! and global_asm! sym operands are not yet supported", + ); } + + let instance = Instance::mono(tcx, def_id); + let symbol = tcx.symbol_name(instance); + global_asm.push_str(symbol.name); } } } } + } - global_asm.push('\n'); - if is_x86 { - global_asm.push_str(".att_syntax\n\n"); - } - } else { - bug!("Expected GlobalAsm found {:?}", item); + global_asm.push('\n'); + if is_x86 { + global_asm.push_str(".att_syntax\n\n"); } } @@ -132,6 +169,7 @@ pub(crate) fn compile_global_asm( config: &GlobalAsmConfig, cgu_name: &str, global_asm: &str, + invocation_temp: Option<&str>, ) -> Result<Option<PathBuf>, String> { if global_asm.is_empty() { return Ok(None); @@ -146,7 +184,7 @@ pub(crate) fn compile_global_asm( global_asm.push('\n'); let global_asm_object_file = add_file_stem_postfix( - config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)), + config.output_filenames.temp_path_for_cgu(OutputType::Object, cgu_name, invocation_temp), ".asm", ); diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 310b226814d..afee5095549 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -161,7 +161,6 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>( stack_slots_input: Vec::new(), stack_slots_output: Vec::new(), stack_slot_size: Size::from_bytes(0), - is_naked: false, }; asm_gen.allocate_registers(); asm_gen.allocate_stack_slots(); @@ -201,114 +200,6 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>( call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs); } -pub(crate) fn codegen_naked_asm<'tcx>( - tcx: TyCtxt<'tcx>, - cx: &mut crate::CodegenCx, - module: &mut dyn Module, - instance: Instance<'tcx>, - span: Span, - symbol_name: &str, - template: &[InlineAsmTemplatePiece], - operands: &[InlineAsmOperand<'tcx>], - options: InlineAsmOptions, -) { - // FIXME add .eh_frame unwind info directives - - let operands = operands - .iter() - .map(|operand| match *operand { - InlineAsmOperand::In { .. } - | InlineAsmOperand::Out { .. } - | InlineAsmOperand::InOut { .. } => { - span_bug!(span, "invalid operand type for naked asm") - } - InlineAsmOperand::Const { ref value } => { - let cv = instance.instantiate_mir_and_normalize_erasing_regions( - tcx, - ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(value.const_), - ); - let const_value = cv - .eval(tcx, ty::TypingEnv::fully_monomorphized(), value.span) - .expect("erroneous constant missed by mono item collection"); - - let value = rustc_codegen_ssa::common::asm_const_to_str( - tcx, - span, - const_value, - FullyMonomorphizedLayoutCx(tcx).layout_of(cv.ty()), - ); - CInlineAsmOperand::Const { value } - } - InlineAsmOperand::SymFn { ref value } => { - if cfg!(not(feature = "inline_asm_sym")) { - tcx.dcx() - .span_err(span, "asm! and global_asm! sym operands are not yet supported"); - } - - let const_ = instance.instantiate_mir_and_normalize_erasing_regions( - tcx, - ty::TypingEnv::fully_monomorphized(), - ty::EarlyBinder::bind(value.const_), - ); - if let ty::FnDef(def_id, args) = *const_.ty().kind() { - let instance = ty::Instance::resolve_for_fn_ptr( - tcx, - ty::TypingEnv::fully_monomorphized(), - def_id, - args, - ) - .unwrap(); - let symbol = tcx.symbol_name(instance); - - // Pass a wrapper rather than the function itself as the function itself may not - // be exported from the main codegen unit and may thus be unreachable from the - // object file created by an external assembler. - let wrapper_name = format!( - "__inline_asm_{}_wrapper_n{}", - cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), - cx.inline_asm_index - ); - cx.inline_asm_index += 1; - let sig = - get_function_sig(tcx, module.target_config().default_call_conv, instance); - create_wrapper_function(module, sig, &wrapper_name, symbol.name); - - CInlineAsmOperand::Symbol { symbol: wrapper_name } - } else { - span_bug!(span, "invalid type for asm sym (fn)"); - } - } - InlineAsmOperand::SymStatic { def_id } => { - assert!(tcx.is_static(def_id)); - let instance = Instance::mono(tcx, def_id); - CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() } - } - InlineAsmOperand::Label { .. } => { - span_bug!(span, "asm! label operands are not yet supported"); - } - }) - .collect::<Vec<_>>(); - - let asm_gen = InlineAssemblyGenerator { - tcx, - arch: tcx.sess.asm_arch.unwrap(), - enclosing_def_id: instance.def_id(), - template, - operands: &operands, - options, - registers: Vec::new(), - stack_slots_clobber: Vec::new(), - stack_slots_input: Vec::new(), - stack_slots_output: Vec::new(), - stack_slot_size: Size::from_bytes(0), - is_naked: true, - }; - - let generated_asm = asm_gen.generate_asm_wrapper(symbol_name); - cx.global_asm.push_str(&generated_asm); -} - struct InlineAssemblyGenerator<'a, 'tcx> { tcx: TyCtxt<'tcx>, arch: InlineAsmArch, @@ -321,13 +212,10 @@ struct InlineAssemblyGenerator<'a, 'tcx> { stack_slots_input: Vec<Option<Size>>, stack_slots_output: Vec<Option<Size>>, stack_slot_size: Size, - is_naked: bool, } impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { fn allocate_registers(&mut self) { - assert!(!self.is_naked); - let sess = self.tcx.sess; let map = allocatable_registers( self.arch, @@ -451,8 +339,6 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { } fn allocate_stack_slots(&mut self) { - assert!(!self.is_naked); - let mut slot_size = Size::from_bytes(0); let mut slots_clobber = vec![None; self.operands.len()]; let mut slots_input = vec![None; self.operands.len()]; @@ -582,36 +468,44 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { if is_x86 { generated_asm.push_str(".intel_syntax noprefix\n"); } - if !self.is_naked { - Self::prologue(&mut generated_asm, self.arch); - - // Save clobbered registers - if !self.options.contains(InlineAsmOptions::NORETURN) { - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_clobber.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::save_register(&mut generated_asm, self.arch, reg, slot); - } - } - // Write input registers + Self::prologue(&mut generated_asm, self.arch); + + // Save clobbered registers + if !self.options.contains(InlineAsmOptions::NORETURN) { for (reg, slot) in self .registers .iter() - .zip(self.stack_slots_input.iter().copied()) + .zip(self.stack_slots_clobber.iter().copied()) .filter_map(|(r, s)| r.zip(s)) { - Self::restore_register(&mut generated_asm, self.arch, reg, slot); + Self::save_register(&mut generated_asm, self.arch, reg, slot); } } + // Write input registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_input.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::restore_register(&mut generated_asm, self.arch, reg, slot); + } + if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { generated_asm.push_str(".att_syntax\n"); } + if self.arch == InlineAsmArch::AArch64 { + for feature in &self.tcx.codegen_fn_attrs(self.enclosing_def_id).target_features { + if feature.name == sym::neon { + continue; + } + writeln!(generated_asm, ".arch_extension {}", feature.name).unwrap(); + } + } + // The actual inline asm for piece in self.template { match piece { @@ -652,6 +546,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { .emit(&mut generated_asm, InlineAsmArch::X86_64, *modifier) .unwrap(), }, + InlineAsmArch::AArch64 => match reg { + InlineAsmReg::AArch64(reg) if reg.vreg_index().is_some() => { + // rustc emits v0 rather than q0 + reg.emit( + &mut generated_asm, + InlineAsmArch::AArch64, + Some(modifier.unwrap_or('q')), + ) + .unwrap() + } + _ => reg + .emit(&mut generated_asm, InlineAsmArch::AArch64, *modifier) + .unwrap(), + }, _ => reg.emit(&mut generated_asm, self.arch, *modifier).unwrap(), } } @@ -665,36 +573,43 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { } generated_asm.push('\n'); + if self.arch == InlineAsmArch::AArch64 { + for feature in &self.tcx.codegen_fn_attrs(self.enclosing_def_id).target_features { + if feature.name == sym::neon { + continue; + } + writeln!(generated_asm, ".arch_extension no{}", feature.name).unwrap(); + } + } + if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) { generated_asm.push_str(".intel_syntax noprefix\n"); } - if !self.is_naked { - if !self.options.contains(InlineAsmOptions::NORETURN) { - // Read output registers - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_output.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::save_register(&mut generated_asm, self.arch, reg, slot); - } - - // Restore clobbered registers - for (reg, slot) in self - .registers - .iter() - .zip(self.stack_slots_clobber.iter().copied()) - .filter_map(|(r, s)| r.zip(s)) - { - Self::restore_register(&mut generated_asm, self.arch, reg, slot); - } + if !self.options.contains(InlineAsmOptions::NORETURN) { + // Read output registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_output.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::save_register(&mut generated_asm, self.arch, reg, slot); + } - Self::epilogue(&mut generated_asm, self.arch); - } else { - Self::epilogue_noreturn(&mut generated_asm, self.arch); + // Restore clobbered registers + for (reg, slot) in self + .registers + .iter() + .zip(self.stack_slots_clobber.iter().copied()) + .filter_map(|(r, s)| r.zip(s)) + { + Self::restore_register(&mut generated_asm, self.arch, reg, slot); } + + Self::epilogue(&mut generated_asm, self.arch); + } else { + Self::epilogue_noreturn(&mut generated_asm, self.arch); } if is_x86 { @@ -809,7 +724,13 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { } InlineAsmArch::AArch64 => { generated_asm.push_str(" str "); - reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(); + match reg { + InlineAsmReg::AArch64(reg) if reg.vreg_index().is_some() => { + // rustc emits v0 rather than q0 + reg.emit(generated_asm, InlineAsmArch::AArch64, Some('q')).unwrap() + } + _ => reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(), + } writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap(); } InlineAsmArch::RiscV64 => { @@ -851,7 +772,13 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { } InlineAsmArch::AArch64 => { generated_asm.push_str(" ldr "); - reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(); + match reg { + InlineAsmReg::AArch64(reg) if reg.vreg_index().is_some() => { + // rustc emits v0 rather than q0 + reg.emit(generated_asm, InlineAsmArch::AArch64, Some('q')).unwrap() + } + _ => reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap(), + } writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap(); } InlineAsmArch::RiscV64 => { diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index 720a0d8fbf5..eb0dfbb69c3 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -54,6 +54,14 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( ); } + "llvm.fptosi.sat.v4i32.v4f32" => { + intrinsic_args!(fx, args => (a); intrinsic); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().fcvt_to_sint_sat(types::I32, lane) + }); + } + _ => { fx.tcx .dcx() diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs index 4c59c81296b..387c87d123a 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs @@ -1,5 +1,9 @@ //! Emulate AArch64 LLVM intrinsics +use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_target::asm::*; + +use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner}; use crate::intrinsics::*; use crate::prelude::*; @@ -17,7 +21,7 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( fx.bcx.ins().fence(); } - "llvm.aarch64.neon.ld1x4.v16i8.p0i8" => { + "llvm.aarch64.neon.ld1x4.v16i8.p0" => { intrinsic_args!(fx, args => (ptr); intrinsic); let ptr = ptr.load_scalar(fx); @@ -49,6 +53,121 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( }); } + "llvm.aarch64.neon.fcvtns.v4i32.v4f32" => { + intrinsic_args!(fx, args => (a); intrinsic); + + // Note: Using inline asm instead of fcvt_to_sint as the latter rounds to zero rather than to nearest + + let a_ptr = a.force_stack(fx).0.get_addr(fx); + let res_place = CPlace::new_stack_slot(fx, ret.layout()); + let res_ptr = res_place.to_ptr().get_addr(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "ldr q0, [x0] + fcvtns v0.4s, v0.4s + str q0, [x1]" + .into(), + )], + &[ + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + value: a_ptr, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: res_ptr, + }, + ], + InlineAsmOptions::NOSTACK, + ); + let res = res_place.to_cvalue(fx); + ret.write_cvalue_transmute(fx, res); + } + + "llvm.aarch64.neon.frecpe.v4f32" => { + intrinsic_args!(fx, args => (a); intrinsic); + + let a_ptr = a.force_stack(fx).0.get_addr(fx); + let res_place = CPlace::new_stack_slot(fx, ret.layout()); + let res_ptr = res_place.to_ptr().get_addr(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "ldr q0, [x0] + frecpe v0.4s, v0.4s + str q0, [x1]" + .into(), + )], + &[ + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + value: a_ptr, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: res_ptr, + }, + ], + InlineAsmOptions::NOSTACK, + ); + let res = res_place.to_cvalue(fx); + ret.write_cvalue_transmute(fx, res); + } + + "llvm.aarch64.neon.frecps.v4f32" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a_ptr = a.force_stack(fx).0.get_addr(fx); + let b_ptr = b.force_stack(fx).0.get_addr(fx); + let res_place = CPlace::new_stack_slot(fx, ret.layout()); + let res_ptr = res_place.to_ptr().get_addr(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "ldr q0, [x0] + ldr q1, [x1] + frecps v0.4s, v0.4s, v1.4s + str q0, [x2]" + .into(), + )], + &[ + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + value: a_ptr, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: b_ptr, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x2, + )), + value: res_ptr, + }, + ], + InlineAsmOptions::NOSTACK, + ); + let res = res_place.to_cvalue(fx); + ret.write_cvalue_transmute(fx, res); + } + _ if intrinsic.starts_with("llvm.aarch64.neon.sqadd.v") || intrinsic.starts_with("llvm.aarch64.neon.uqadd.v") => { @@ -134,7 +253,7 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( } let res = CValue::by_val( fx.bcx.ins().uextend(types::I32, res_val), - fx.layout_of(fx.tcx.types.u32), + fx.layout_of(fx.tcx.types.i32), ); ret.write_cvalue(fx, res); } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 75f3a3c1972..d3f47ad7263 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -1,5 +1,4 @@ -//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, -//! functions marked with the `#[rustc_intrinsic]` attribute +//! Codegen of intrinsics. This includes functions marked with the `#[rustc_intrinsic]` attribute //! and LLVM intrinsics that have symbol names starting with `llvm.`. macro_rules! intrinsic_args { diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index ab3386a9b4c..ab09a6f8b38 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -41,9 +41,8 @@ use std::sync::Arc; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; -use rustc_codegen_ssa::CodegenResults; -use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::traits::CodegenBackend; +use rustc_codegen_ssa::{CodegenResults, TargetConfig}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -125,6 +124,7 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> { /// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). struct CodegenCx { output_filenames: Arc<OutputFilenames>, + invocation_temp: Option<String>, should_write_ir: bool, global_asm: String, inline_asm_index: usize, @@ -143,6 +143,7 @@ impl CodegenCx { }; CodegenCx { output_filenames: tcx.output_filenames(()).clone(), + invocation_temp: tcx.sess.invocation_temp.clone(), should_write_ir: crate::pretty_clif::should_write_ir(tcx), global_asm: String::new(), inline_asm_index: 0, @@ -177,7 +178,7 @@ impl CodegenBackend for CraneliftCodegenBackend { } } - fn target_features_cfg(&self, sess: &Session) -> (Vec<Symbol>, Vec<Symbol>) { + fn target_config(&self, sess: &Session) -> TargetConfig { // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)] let target_features = if sess.target.arch == "x86_64" && sess.target.os != "none" { // x86_64 mandates SSE2 support and rustc requires the x87 feature to be enabled @@ -196,7 +197,16 @@ impl CodegenBackend for CraneliftCodegenBackend { }; // FIXME do `unstable_target_features` properly let unstable_target_features = target_features.clone(); - (target_features, unstable_target_features) + + TargetConfig { + target_features, + unstable_target_features, + // Cranelift does not yet support f16 or f128 + has_reliable_f16: false, + has_reliable_f16_math: false, + has_reliable_f128: false, + has_reliable_f128_math: false, + } } fn print_version(&self) { @@ -214,15 +224,14 @@ impl CodegenBackend for CraneliftCodegenBackend { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.dcx().fatal(err)) }); - match config.codegen_mode { - CodegenMode::Aot => driver::aot::run_aot(tcx, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - #[cfg(feature = "jit")] - driver::jit::run_jit(tcx, config.codegen_mode, config.jit_args); - - #[cfg(not(feature = "jit"))] - tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } + if config.jit_mode { + #[cfg(feature = "jit")] + driver::jit::run_jit(tcx, config.jit_args); + + #[cfg(not(feature = "jit"))] + tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } else { + driver::aot::run_aot(tcx, metadata, need_metadata_module) } } @@ -247,21 +256,19 @@ fn enable_verifier(sess: &Session) -> bool { } fn target_triple(sess: &Session) -> target_lexicon::Triple { - // FIXME(madsmtm): Use `sess.target.llvm_target` once target-lexicon supports unversioned macOS. - // See <https://github.com/bytecodealliance/target-lexicon/pull/113> - match versioned_llvm_target(sess).parse() { + match sess.target.llvm_target.parse() { Ok(triple) => triple, Err(err) => sess.dcx().fatal(format!("target not recognized: {}", err)), } } -fn build_isa(sess: &Session) -> Arc<dyn TargetIsa + 'static> { +fn build_isa(sess: &Session, jit: bool) -> Arc<dyn TargetIsa + 'static> { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); let mut flags_builder = settings::builder(); - flags_builder.enable("is_pic").unwrap(); + flags_builder.set("is_pic", if jit { "false" } else { "true" }).unwrap(); let enable_verifier = if enable_verifier(sess) { "true" } else { "false" }; flags_builder.set("enable_verifier", enable_verifier).unwrap(); flags_builder.set("regalloc_checker", enable_verifier).unwrap(); diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index 6d5df2b0043..6eef97c14dd 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -1,6 +1,6 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use rustc_hir::LangItem; -use rustc_middle::ty::{AssocKind, GenericArg}; +use rustc_middle::ty::{AssocTag, GenericArg}; use rustc_session::config::EntryFnType; use rustc_span::{DUMMY_SP, Ident}; @@ -104,10 +104,10 @@ pub(crate) fn maybe_create_entry_wrapper( let termination_trait = tcx.require_lang_item(LangItem::Termination, None); let report = tcx .associated_items(termination_trait) - .find_by_name_and_kind( + .find_by_ident_and_kind( tcx, Ident::from_str("report"), - AssocKind::Fn, + AssocTag::Fn, termination_trait, ) .unwrap(); diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index f44e2459a78..2a4d1e3ae57 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -395,8 +395,12 @@ pub(crate) fn codegen_ptr_binop<'tcx>( in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx> { - let is_thin_ptr = - in_lhs.layout().ty.builtin_deref(true).map(|ty| !has_ptr_meta(fx.tcx, ty)).unwrap_or(true); + let is_thin_ptr = in_lhs + .layout() + .ty + .builtin_deref(true) + .map(|ty| !fx.tcx.type_has_metadata(ty, ty::TypingEnv::fully_monomorphized())) + .unwrap_or(true); if is_thin_ptr { match bin_op { diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index cc739fefcd0..f8a19589fdd 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -746,7 +746,7 @@ impl<'tcx> CPlace<'tcx> { }; let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); - if has_ptr_meta(fx.tcx, field_layout.ty) { + if fx.tcx.type_has_metadata(field_layout.ty, ty::TypingEnv::fully_monomorphized()) { CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) } else { CPlace::for_ptr(field_ptr, field_layout) @@ -832,7 +832,7 @@ impl<'tcx> CPlace<'tcx> { pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> { let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap()); - if has_ptr_meta(fx.tcx, inner_layout.ty) { + if fx.tcx.type_has_metadata(inner_layout.ty, ty::TypingEnv::fully_monomorphized()) { let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) } else { @@ -845,7 +845,7 @@ impl<'tcx> CPlace<'tcx> { fx: &mut FunctionCx<'_, '_, 'tcx>, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - if has_ptr_meta(fx.tcx, self.layout().ty) { + if fx.tcx.type_has_metadata(self.layout().ty, ty::TypingEnv::fully_monomorphized()) { let (ptr, extra) = self.to_ptr_unsized(); CValue::by_val_pair(ptr.get_addr(fx), extra, layout) } else { |