diff options
| -rw-r--r-- | .vscode/settings.json | 2 | ||||
| -rw-r--r-- | src/abi/mod.rs | 8 | ||||
| -rw-r--r-- | src/allocator.rs | 4 | ||||
| -rw-r--r-- | src/base.rs | 28 | ||||
| -rw-r--r-- | src/common.rs | 12 | ||||
| -rw-r--r-- | src/config.rs | 6 | ||||
| -rw-r--r-- | src/constant.rs | 31 | ||||
| -rw-r--r-- | src/debuginfo/unwind.rs | 66 | ||||
| -rw-r--r-- | src/driver/aot.rs | 25 | ||||
| -rw-r--r-- | src/driver/jit.rs | 195 | ||||
| -rw-r--r-- | src/driver/mod.rs | 54 | ||||
| -rw-r--r-- | src/inline_asm.rs | 3 | ||||
| -rw-r--r-- | src/intrinsics/mod.rs | 10 | ||||
| -rw-r--r-- | src/lib.rs | 43 | ||||
| -rw-r--r-- | src/main_shim.rs | 28 | ||||
| -rw-r--r-- | src/metadata.rs | 12 | ||||
| -rw-r--r-- | src/trap.rs | 3 | ||||
| -rw-r--r-- | src/value_and_place.rs | 2 | ||||
| -rw-r--r-- | src/vtable.rs | 12 |
19 files changed, 255 insertions, 289 deletions
diff --git a/.vscode/settings.json b/.vscode/settings.json index 0cd576e160f..9009a532c54 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this - "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"], + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], "rust-analyzer.assist.importMergeBehavior": "last", "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ diff --git a/src/abi/mod.rs b/src/abi/mod.rs index f898348f97b..f450f366787 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -71,8 +71,8 @@ pub(crate) fn import_function<'tcx>( impl<'tcx> FunctionCx<'_, '_, 'tcx> { /// Instance must be monomorphized pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { - let func_id = import_function(self.tcx, self.cx.module, inst); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); if self.clif_comments.enabled() { self.add_comment(func_ref, format!("{:?}", inst)); @@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = self.module.declare_function(&name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); diff --git a/src/allocator.rs b/src/allocator.rs index f60645a9f97..a09e3257786 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -11,7 +11,7 @@ use rustc_span::symbol::sym; pub(crate) fn codegen( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) -> bool { let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; @@ -29,7 +29,7 @@ pub(crate) fn codegen( fn codegen_inner( module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, kind: AllocatorKind, ) { let usize_ty = module.target_config().pointer_type(); diff --git a/src/base.rs b/src/base.rs index 3a5b2be462a..069166f7b68 100644 --- a/src/base.rs +++ b/src/base.rs @@ -9,7 +9,11 @@ use rustc_target::abi::call::FnAbi; use crate::constant::ConstantCx; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { +pub(crate) fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { let tcx = cx.tcx; let _inst_guard = @@ -20,8 +24,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Declare function let symbol_name = tcx.symbol_name(instance); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); + let sig = get_function_sig(tcx, module.isa().triple(), instance); + let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -40,11 +44,12 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); let mut fx = FunctionCx { cx, + module, tcx, pointer_type, vtables: FxHashMap::default(), @@ -94,7 +99,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let source_info_set = fx.source_info_set; let local_map = fx.local_map; - fx.constants_cx.finalize(fx.tcx, &mut *fx.cx.module); + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); // Store function in context let context = &mut cx.cached_context; @@ -114,8 +119,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // instruction, which doesn't have an encoding. context.compute_cfg(); context.compute_domtree(); - context.eliminate_unreachable_code(cx.module.isa()).unwrap(); - context.dce(cx.module.isa()).unwrap(); + context.eliminate_unreachable_code(module.isa()).unwrap(); + context.dce(module.isa()).unwrap(); // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't // invalidate it when it would change. context.domtree.clear(); @@ -123,7 +128,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In context.want_disasm = crate::pretty_clif::should_write_ir(tcx); // Define function - let module = &mut cx.module; tcx.sess.time("define function", || { module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) @@ -134,7 +138,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In crate::pretty_clif::write_clif_file( tcx, "opt", - Some(cx.module.isa()), + Some(module.isa()), instance, &context, &clif_comments, @@ -149,7 +153,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In } // Define debuginfo for function - let isa = cx.module.isa(); + let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; tcx.sess.time("generate debug info", || { @@ -654,7 +658,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -834,7 +838,7 @@ fn codegen_stmt<'tcx>( let elem_size: u64 = pointee.size.bytes(); let bytes = if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes); } } } diff --git a/src/common.rs b/src/common.rs index 6752bb42dc8..92e4435565e 100644 --- a/src/common.rs +++ b/src/common.rs @@ -228,8 +228,9 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { - pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>, @@ -341,7 +342,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn triple(&self) -> &target_lexicon::Triple { - self.cx.module.isa().triple() + self.module.isa().triple() } pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { @@ -354,15 +355,14 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); let msg_id = self - .cx .module .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) .unwrap(); // Ignore DuplicateDefinition error, as the data will be the same - let _ = self.cx.module.define_data(msg_id, &data_ctx); + let _ = self.module.define_data(msg_id, &data_ctx); - let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } diff --git a/src/config.rs b/src/config.rs index 0cf02e2bbb5..e59a0cb0a23 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,10 +5,14 @@ fn bool_env_var(key: &str) -> bool { env::var(key).as_ref().map(|val| &**val) == Ok("1") } +/// The mode to use for compilation. #[derive(Copy, Clone, Debug)] pub enum CodegenMode { + /// AOT compile the crate. This is the default. Aot, + /// JIT compile and execute the crate. Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. JitLazy, } @@ -25,6 +29,7 @@ impl FromStr for CodegenMode { } } +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. #[derive(Clone, Debug)] pub struct BackendConfig { /// Should the crate be AOT compiled or JIT executed. @@ -76,6 +81,7 @@ impl Default for BackendConfig { } impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. pub fn from_opts(opts: &[String]) -> Result<Self, String> { fn parse_bool(name: &str, value: &str) -> Result<bool, String> { value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) diff --git a/src/constant.rs b/src/constant.rs index 339580eb8f4..0a0e02d2639 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -13,7 +13,7 @@ use rustc_middle::ty::ConstKind; use cranelift_codegen::ir::GlobalValueData; use cranelift_module::*; -use crate::{prelude::*, CodegenCx}; +use crate::prelude::*; pub(crate) struct ConstantCx { todo: Vec<TodoItem>, @@ -78,10 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(cx: &mut CodegenCx<'_, '_>, def_id: DefId) { +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); - constants_cx.finalize(cx.tcx, &mut *cx.module); + constants_cx.finalize(tcx, module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -89,8 +89,8 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("tls {:?}", def_id)); } @@ -103,8 +103,8 @@ fn codegen_static_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -191,29 +191,28 @@ pub(crate) fn codegen_const_value<'tcx>( fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); let data_id = data_id_for_alloc_id( &mut fx.constants_cx, - fx.cx.module, + fx.module, ptr.alloc_id, alloc.mutability, ); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { - let func_id = - crate::abi::import_function(fx.tcx, fx.cx.module, instance); + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); let local_func_id = - fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) } Some(GlobalAlloc::Static(def_id)) => { assert!(fx.tcx.is_static(def_id)); - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -255,9 +254,9 @@ fn pointer_for_allocation<'tcx>( let alloc_id = fx.tcx.create_memory_alloc(alloc); fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); let data_id = - data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.cx.module, alloc_id, alloc.mutability); + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", alloc_id)); } diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 357c9fe6ed8..ca7083cccb8 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -5,17 +5,19 @@ use crate::prelude::*; use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; use crate::backend::WriteDebugInfo; -pub(crate) struct UnwindContext<'tcx> { - tcx: TyCtxt<'tcx>, +pub(crate) struct UnwindContext { + endian: RunTimeEndian, frame_table: FrameTable, cie_id: Option<CieId>, } -impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { +impl UnwindContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = super::target_endian(tcx); let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { @@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> { None }; - UnwindContext { tcx, frame_table, cie_id } + UnwindContext { endian, frame_table, cie_id } } pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { @@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> { } pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if !eh_frame.0.writer.slice().is_empty() { @@ -71,16 +72,12 @@ impl<'tcx> UnwindContext<'tcx> { } #[cfg(feature = "jit")] - pub(crate) unsafe fn register_jit( - self, - jit_module: &cranelift_jit::JITModule, - ) -> Option<UnwindRegistry> { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { - return None; + return; } let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); @@ -88,7 +85,10 @@ impl<'tcx> UnwindContext<'tcx> { // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); - let mut registrations = Vec::new(); + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + #[allow(unused_variables)] + let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame); // ======================================================================= // Everything after this line up to the end of the file is loosly based on @@ -96,8 +96,8 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(target_os = "macos")] { // On macOS, `__register_frame` takes a pointer to a single FDE - let start = eh_frame.as_ptr(); - let end = start.add(eh_frame.len()); + let start = eh_frame; + let end = start.add(eh_frame_len); let mut current = start; // Walk all of the entries in the frame table and register them @@ -107,7 +107,6 @@ impl<'tcx> UnwindContext<'tcx> { // Skip over the CIE if current != start { __register_frame(current); - registrations.push(current as usize); } // Move to the next table entry (+4 because the length itself is not inclusive) @@ -117,41 +116,12 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(not(target_os = "macos"))] { // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = eh_frame.as_ptr(); - __register_frame(ptr); - registrations.push(ptr as usize); + __register_frame(eh_frame); } - - Some(UnwindRegistry { _frame_table: eh_frame, registrations }) } } -/// Represents a registry of function unwind information for System V ABI. -pub(crate) struct UnwindRegistry { - _frame_table: Vec<u8>, - registrations: Vec<usize>, -} - extern "C" { // libunwind import fn __register_frame(fde: *const u8); - fn __deregister_frame(fde: *const u8); -} - -impl Drop for UnwindRegistry { - fn drop(&mut self) { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 1f731a756ed..3ff96d710e5 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -31,7 +31,7 @@ fn emit_module( kind: ModuleKind, module: ObjectModule, debug: Option<DebugContext<'_>>, - unwind_context: UnwindContext<'_>, + unwind_context: UnwindContext, ) -> ModuleCodegenResult { let mut product = module.finish(); @@ -107,21 +107,22 @@ fn module_codegen( let isa = crate::build_isa(tcx.sess, &backend_config); let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); let mut cx = crate::CodegenCx::new( tcx, backend_config.clone(), - &mut module, + module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, ); - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx + .sess + .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx, def_id), + MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { @@ -133,9 +134,7 @@ fn module_codegen( } } } - let (global_asm, debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); let codegen_result = emit_module( tcx, @@ -143,16 +142,16 @@ fn module_codegen( cgu.name().as_str().to_string(), ModuleKind::Regular, module, - debug, - unwind_context, + cx.debug_context, + cx.unwind_context, ); - codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); codegen_result } -pub(super) fn run_aot( +pub(crate) fn run_aot( tcx: TyCtxt<'_>, backend_config: BackendConfig, metadata: EncodedMetadata, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index ffe449115b9..63ac02f83b9 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -1,4 +1,4 @@ -//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. use std::cell::RefCell; @@ -8,32 +8,62 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; -thread_local! { - pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None); - pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); +struct JitState { + backend_config: BackendConfig, + jit_module: JITModule, } -pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { - if !tcx.sess.opts.output_types.should_codegen() { - tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); - } +thread_local! { + static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None); +} +fn create_jit_module<'tcx>( + tcx: TyCtxt<'tcx>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (JITModule, CodegenCx<'tcx>) { let imported_symbols = load_imported_symbols_for_jit(tcx); - let isa = crate::build_isa(tcx.sess, &backend_config); + let isa = crate::build_isa(tcx.sess, backend_config); let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); - jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); + jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); - assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); + + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); + + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + ); + + (jit_module, cx) +} + +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.sess.fatal("can't jit non-executable crate"); + } + + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); let mono_items = cgus @@ -44,44 +74,38 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::<Vec<(_, (_, _))>>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), &mut jit_module, false); - super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx, def_id); + crate::constant::codegen_static(tcx, &mut jit_module, def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); + let item = tcx.hir().item(item_id); tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); } } } }); - let (global_asm, _debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - jit_module.finalize_definitions(); - - if !global_asm.is_empty() { + if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); - tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; + unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" @@ -97,61 +121,27 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); - BACKEND_CONFIG.with(|tls_backend_config| { - assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); - let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx); - - match entry_ty { - EntryFnType::Main => { - // FIXME set program arguments somehow - - let main_sig = Signature { - params: vec![], - returns: vec![], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) - .unwrap(); - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; - f(); - std::process::exit(0); - } - EntryFnType::Start => { - let start_sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new( - jit_module.target_config().pointer_type(), /*isize*/ - )], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let start_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) - .unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_start) }; - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - } - } + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } #[no_mangle] @@ -160,24 +150,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 // lift is used to ensure the correct lifetime for instance. let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - CURRENT_MODULE.with(|jit_module| { - let mut jit_module = jit_module.borrow_mut(); - let jit_module = jit_module.as_mut().unwrap(); - let backend_config = - BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + let backend_config = lazy_jit_state.backend_config.clone(); let name = tcx.symbol_name(instance).name.to_string(); let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); + let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); - let (global_asm, _debug_context, unwind_context) = cx.finalize(); - assert!(global_asm.is_empty()); + assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + unsafe { cx.unwind_context.register_jit(&jit_module) }; jit_module.get_finalized_function(func_id) }) }) @@ -247,35 +236,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { let tcx = cx.tcx; - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); - let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); + let func_id = module.declare_function(&name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); - let jit_fn = cx - .module + let jit_fn = module .declare_function( "__clif_jit_fn", Linkage::Import, &Signature { - call_conv: cx.module.target_config().default_call_conv, + call_conv: module.target_config().default_call_conv, params: vec![AbiParam::new(pointer_type)], returns: vec![AbiParam::new(pointer_type)], }, ) .unwrap(); - let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + cx.cached_context.clear(); + let trampoline = &mut cx.cached_context.func; + trampoline.signature = sig.clone(); + let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); let entry_block = trampoline_builder.create_block(); @@ -290,10 +281,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - cx.module + module .define_function( func_id, - &mut Context::for_function(trampoline), + &mut cx.cached_context, &mut NullTrapSink {}, &mut NullStackMapSink {}, ) diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 960e10182d8..393cde3e9ad 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -1,63 +1,37 @@ -//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions -//! like JIT executing or writing object files. +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static -use std::any::Any; - -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; -use crate::CodegenMode; -mod aot; +pub(crate) mod aot; #[cfg(feature = "jit")] -mod jit; - -pub(crate) fn codegen_crate( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - backend_config: crate::BackendConfig, -) -> Box<dyn Any> { - tcx.sess.abort_if_errors(); - - match backend_config.codegen_mode { - CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - let is_executable = - tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } - - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx, backend_config); - - #[cfg(not(feature = "jit"))] - tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } - } -} +pub(crate) mod jit; fn predefine_mono_items<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { - cx.tcx.sess.time("predefine functions", || { - let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); + tcx.sess.time("predefine functions", || { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = cx.tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name.to_string(); let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); - let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( mono_item, linkage, visibility, is_compiler_builtins, ); - cx.module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(&name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 7187bc785e4..669a6d35075 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -201,7 +201,6 @@ fn call_inline_asm<'tcx>( } let inline_asm_func = fx - .cx .module .declare_function( asm_name, @@ -213,7 +212,7 @@ fn call_inline_asm<'tcx>( }, ) .unwrap(); - let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(inline_asm_func, asm_name); } diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 58a24c41930..d08271f853b 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -498,10 +498,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( if intrinsic.contains("nonoverlapping") { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -517,10 +517,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} if intrinsic.contains("nonoverlapping") { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; size_of_val, <T> (c ptr) { @@ -670,7 +670,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, <T> (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. diff --git a/src/lib.rs b/src/lib.rs index f94c9c1dab5..5a75b9be0cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] @@ -33,6 +33,7 @@ use rustc_middle::ty::query::Providers; use rustc_session::config::OutputFilenames; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; pub use crate::config::*; @@ -118,42 +119,36 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> { } } -struct CodegenCx<'m, 'tcx: 'm> { +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). +struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, - module: &'m mut dyn Module, global_asm: String, cached_context: Context, debug_context: Option<DebugContext<'tcx>>, - unwind_context: UnwindContext<'tcx>, + unwind_context: UnwindContext, } -impl<'m, 'tcx> CodegenCx<'m, 'tcx> { +impl<'tcx> CodegenCx<'tcx> { fn new( tcx: TyCtxt<'tcx>, backend_config: BackendConfig, - module: &'m mut dyn Module, + isa: &dyn TargetIsa, debug_info: bool, ) -> Self { - let unwind_context = UnwindContext::new( - tcx, - module.isa(), - matches!(backend_config.codegen_mode, CodegenMode::Aot), - ); - let debug_context = - if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None }; + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + + let unwind_context = + UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); + let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; CodegenCx { tcx, - module, global_asm: String::new(), cached_context: Context::new(), debug_context, unwind_context, } } - - fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) { - (self.global_asm, self.debug_context, self.unwind_context) - } } pub struct CraneliftCodegenBackend { @@ -186,13 +181,23 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box<dyn Any> { + tcx.sess.abort_if_errors(); let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - driver::codegen_crate(tcx, metadata, need_metadata_module, config) + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + let _: ! = driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } } fn join_codegen( diff --git a/src/main_shim.rs b/src/main_shim.rs index a6266f50776..4beb6c65a77 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -9,9 +9,10 @@ use crate::prelude::*; pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, + is_jit: bool, ) { - let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( def_id.to_def_id(), match entry_ty { @@ -23,18 +24,19 @@ pub(crate) fn maybe_create_entry_wrapper( }; let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, rust_main_def_id: DefId, - use_start_lang_item: bool, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -74,7 +76,12 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); - let call_inst = if use_start_lang_item { + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + bcx.ins().call(main_func_ref, &[]); + bcx.ins().iconst(m.target_config().pointer_type(), 0) + } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( tcx, @@ -90,13 +97,14 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] } else { // using user-defined start fn - bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] }; - let result = bcx.inst_results(call_inst)[0]; bcx.ins().return_(&[result]); bcx.seal_all_blocks(); bcx.finalize(); diff --git a/src/metadata.rs b/src/metadata.rs index 7bd9ebbb611..978a1e722dd 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -14,6 +14,18 @@ use rustc_target::spec::Target; use crate::backend::WriteMetadata; +/// The metadata loader used by cg_clif. +/// +/// The metadata is stored in the same format as cg_llvm. +/// +/// # Metadata location +/// +/// <dl> +/// <dt>rlib</dt> +/// <dd>The metadata can be found in the `lib.rmeta` file inside of the ar archive.</dd> +/// <dt>dylib</dt> +/// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd> +/// </dl> pub(crate) struct CraneliftMetadataLoader; fn load_metadata_with( diff --git a/src/trap.rs b/src/trap.rs index 4e52fb16121..819c8b51558 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -4,7 +4,6 @@ use crate::prelude::*; fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { let puts = fx - .cx .module .declare_function( "puts", @@ -16,7 +15,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { }, ) .unwrap(); - let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index b97d3900984..9a572c3501f 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -554,7 +554,7 @@ impl<'tcx> CPlace<'tcx> { let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memory_copy( - fx.cx.module.target_config(), + fx.module.target_config(), to_addr, from_addr, size, diff --git a/src/vtable.rs b/src/vtable.rs index 60bc53024a7..29e960bb84d 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -80,7 +80,7 @@ pub(crate) fn get_vtable<'tcx>( data_id }; - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } @@ -94,7 +94,7 @@ fn build_vtable<'tcx>( let drop_in_place_fn = import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), ); @@ -111,7 +111,7 @@ fn build_vtable<'tcx>( opt_mth.map(|(def_id, substs)| { import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) .unwrap() .polymorphize(fx.tcx), @@ -132,16 +132,16 @@ fn build_vtable<'tcx>( for (i, component) in components.into_iter().enumerate() { if let Some(func_id) = component { - let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr((i * usize_size) as u32, func_ref); } } data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx.cx.module.declare_anonymous_data(false, false).unwrap(); + let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); - fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + fx.module.define_data(data_id, &data_ctx).unwrap(); data_id } |