diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src/driver')
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/driver/aot.rs | 31 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/driver/jit.rs | 166 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/driver/mod.rs | 56 |
3 files changed, 194 insertions, 59 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 78d6ff0cb00..16f9bfc9918 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::cstore::EncodedMetadata; -use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{DebugInfo, OutputType}; @@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege } } - let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); + let mut cx = crate::CodegenCx::new( + tcx, + module, + tcx.sess.opts.debuginfo != DebugInfo::None, + true, + ); super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, (linkage, visibility)) in mono_items { let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + match mono_item { + MonoItem::Fn(inst) => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id) + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { + cx.global_asm.push_str(&*asm.as_str()); + cx.global_asm.push_str("\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } + } + } } let (mut module, global_asm, debug, mut unwind_context) = tcx.sess.time("finalize CodegenCx", || cx.finalize()); @@ -236,7 +259,7 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); - let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa()); + let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 5a844841c2c..9a42c675cc1 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -1,16 +1,23 @@ //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object //! files. +use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; use rustc_codegen_ssa::CrateInfo; +use rustc_middle::mir::mono::MonoItem; -use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; +use cranelift_jit::{JITBuilder, JITModule}; use crate::prelude::*; +use crate::{CodegenCx, CodegenMode}; -pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { +thread_local! { + pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); +} + +pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); } @@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = SimpleJITBuilder::with_isa( - crate::build_isa(tcx.sess, false), + let mut jit_builder = JITBuilder::with_isa( + crate::build_isa(tcx.sess), cranelift_module::default_libcall_names(), ); + jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy)); jit_builder.symbols(imported_symbols); - let mut jit_module = SimpleJITModule::new(jit_builder); + let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); let sig = Signature { @@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { .into_iter() .collect::<Vec<(_, (_, _))>>(); - let mut cx = crate::CodegenCx::new(tcx, jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); - let (mut jit_module, global_asm, _debug, mut unwind_context) = - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); - super::codegen_mono_item(&mut cx, mono_item, linkage); + super::time(tcx, "codegen mono items", || { + super::predefine_mono_items(&mut cx, &mono_items); + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + match mono_item { + MonoItem::Fn(inst) => match codegen_mode { + CodegenMode::Aot => unreachable!(), + CodegenMode::Jit => { + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, inst, linkage) + }); + } + CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + }, + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id); + } + MonoItem::GlobalAsm(hir_id) => { + let item = cx.tcx.hir().expect_item(hir_id); + tcx.sess + .span_fatal(item.span, "Global asm is not supported in JIT mode"); + } } - tcx.sess.time("finalize CodegenCx", || cx.finalize()) - }); + } + }); + + let (mut jit_module, global_asm, _debug, mut unwind_context) = + tcx.sess.time("finalize CodegenCx", || cx.finalize()); + jit_module.finalize_definitions(); + if !global_asm.is_empty() { - tcx.sess.fatal("Global asm is not supported in JIT mode"); + tcx.sess.fatal("Inline asm is not supported in JIT mode"); } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); @@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); + println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_main) }; @@ -107,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); + CURRENT_MODULE + .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + let ret = f(args.len() as c_int, argv.as_ptr()); std::process::exit(ret); } +#[no_mangle] +extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 { + rustc_middle::ty::tls::with(|tcx| { + // lift is used to ensure the correct lifetime for instance. + let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); + + CURRENT_MODULE.with(|jit_module| { + let mut jit_module = jit_module.borrow_mut(); + let jit_module = jit_module.as_mut().unwrap(); + let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false); + + let (name, sig) = crate::abi::get_function_name_and_sig( + tcx, + cx.module.isa().triple(), + instance, + true, + ); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + cx.module.prepare_for_function_redefine(func_id).unwrap(); + + tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, instance, Linkage::Export) + }); + + let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize(); + assert!(global_asm.is_empty()); + jit_module.finalize_definitions(); + std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + jit_module.get_finalized_function(func_id) + }) + }) +} + fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { use rustc_middle::middle::dependency_format::Linkage; @@ -171,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } + +pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) { + let tcx = cx.tcx; + + let pointer_type = cx.module.target_config().pointer_type(); + + let (name, sig) = + crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true); + let func_id = cx + .module + .declare_function(&name, Linkage::Export, &sig) + .unwrap(); + + let instance_ptr = Box::into_raw(Box::new(inst)); + + let jit_fn = cx + .module + .declare_function( + "__clif_jit_fn", + Linkage::Import, + &Signature { + call_conv: cx.module.target_config().default_call_conv, + params: vec![AbiParam::new(pointer_type)], + returns: vec![AbiParam::new(pointer_type)], + }, + ) + .unwrap(); + + let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + let mut builder_ctx = FunctionBuilderContext::new(); + let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + + let jit_fn = cx + .module + .declare_func_in_func(jit_fn, trampoline_builder.func); + let sig_ref = trampoline_builder.func.import_signature(sig); + + let entry_block = trampoline_builder.create_block(); + trampoline_builder.append_block_params_for_function_params(entry_block); + let fn_args = trampoline_builder + .func + .dfg + .block_params(entry_block) + .to_vec(); + + trampoline_builder.switch_to_block(entry_block); + let instance_ptr = trampoline_builder + .ins() + .iconst(pointer_type, instance_ptr as u64 as i64); + let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]); + let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0]; + let call_inst = trampoline_builder + .ins() + .call_indirect(sig_ref, jitted_fn, &fn_args); + let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); + trampoline_builder.ins().return_(&ret_vals); + + cx.module + .define_function( + func_id, + &mut Context::for_function(trampoline), + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index 7b8cc2ddd48..9f4ea9a3865 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; +use crate::CodegenMode; mod aot; #[cfg(feature = "jit")] @@ -20,24 +21,25 @@ pub(crate) fn codegen_crate( ) -> Box<dyn Any> { tcx.sess.abort_if_errors(); - if config.use_jit { - let is_executable = tcx - .sess - .crate_types() - .contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } + match config.codegen_mode { + CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + let is_executable = tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::Executable); + if !is_executable { + tcx.sess.fatal("can't jit non-executable crate"); + } - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx); + #[cfg(feature = "jit")] + let _: ! = jit::run_jit(tcx, config.codegen_mode); - #[cfg(not(feature = "jit"))] - tcx.sess - .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + #[cfg(not(feature = "jit"))] + tcx.sess + .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } } - - aot::run_aot(tcx, metadata, need_metadata_module) } fn predefine_mono_items<'tcx>( @@ -63,30 +65,6 @@ fn predefine_mono_items<'tcx>( }); } -fn codegen_mono_item<'tcx, M: Module>( - cx: &mut crate::CodegenCx<'tcx, M>, - mono_item: MonoItem<'tcx>, - linkage: Linkage, -) { - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage)); - } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id), - MonoItem::GlobalAsm(hir_id) => { - let item = cx.tcx.hir().expect_item(hir_id); - if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { - cx.global_asm.push_str(&*asm.as_str()); - cx.global_asm.push_str("\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); - } - } - } -} - fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { if std::env::var("CG_CLIF_DISPLAY_CG_TIME") .as_ref() |