diff options
| author | Erin Power <erin.power@embark-studios.com> | 2021-04-30 14:49:58 +0200 |
|---|---|---|
| committer | Erin Power <erin.power@embark-studios.com> | 2021-04-30 14:49:58 +0200 |
| commit | 505aa48f24abb36c07bea7f8d4de321de193cc41 (patch) | |
| tree | a6b214063a4bbcdc2c037693314781df215ec2c9 /compiler/rustc_codegen_cranelift/src | |
| parent | 49920bc581743d6edb9f82fbff4cbafebc212619 (diff) | |
| parent | ddd4ce25535cf71203ba3700896131ce55fde795 (diff) | |
| download | rust-505aa48f24abb36c07bea7f8d4de321de193cc41.tar.gz rust-505aa48f24abb36c07bea7f8d4de321de193cc41.zip | |
Sync rustc_codegen_cranelift 'ddd4ce25535cf71203ba3700896131ce55fde795'
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
28 files changed, 691 insertions, 1172 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 0e7829eaa26..54c8fb0e7b8 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -63,16 +63,16 @@ pub(crate) fn import_function<'tcx>( module: &mut dyn Module, inst: Instance<'tcx>, ) -> FuncId { - let name = tcx.symbol_name(inst).name.to_string(); + let name = tcx.symbol_name(inst).name; let sig = get_function_sig(tcx, module.isa().triple(), inst); - module.declare_function(&name, Linkage::Import, &sig).unwrap() + module.declare_function(name, Linkage::Import, &sig).unwrap() } impl<'tcx> FunctionCx<'_, '_, 'tcx> { /// Instance must be monomorphized pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { - let func_id = import_function(self.tcx, self.cx.module, inst); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); if self.clif_comments.enabled() { self.add_comment(func_ref, format!("{:?}", inst)); @@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); @@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ pub(crate) fn codegen_terminator_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span, - current_block: Block, func: &Operand<'tcx>, args: &[Operand<'tcx>], destination: Option<(Place<'tcx>, BasicBlock)>, @@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)) .unwrap_or(false); if is_cold { - fx.cold_blocks.insert(current_block); + // FIXME Mark current_block block as cold once Cranelift supports it } // Unpack arguments tuple for closures diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index f60645a9f97..a09e3257786 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -11,7 +11,7 @@ use rustc_span::symbol::sym; pub(crate) fn codegen( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) -> bool { let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; @@ -29,7 +29,7 @@ pub(crate) fn codegen( fn codegen_inner( module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, kind: AllocatorKind, ) { let usize_ty = module.target_config().pointer_type(); diff --git a/compiler/rustc_codegen_cranelift/src/backend.rs b/compiler/rustc_codegen_cranelift/src/backend.rs index eb7927fc4ad..05c06bac27d 100644 --- a/compiler/rustc_codegen_cranelift/src/backend.rs +++ b/compiler/rustc_codegen_cranelift/src/backend.rs @@ -5,23 +5,23 @@ use std::convert::{TryFrom, TryInto}; use rustc_data_structures::fx::FxHashMap; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_module::FuncId; +use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; use object::write::*; use object::{RelocationEncoding, SectionKind, SymbolFlags}; -use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; - use gimli::SectionId; use crate::debuginfo::{DebugReloc, DebugRelocName}; pub(crate) trait WriteMetadata { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, is_like_osx: bool); + fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>); } impl WriteMetadata for object::write::Object { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>) { let segment = self.segment_name(object::write::StandardSegment::Data).to_vec(); let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); let offset = self.append_section_data(section_id, &data, 1); @@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> { - let triple = crate::build_isa(sess).triple().clone(); + let triple = crate::target_triple(sess); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -141,13 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object metadata_object.write().unwrap() } -pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { - let mut builder = ObjectBuilder::new( - crate::build_isa(sess), - name + ".o", - cranelift_module::default_libcall_names(), - ) - .unwrap(); +pub(crate) fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule { + let mut builder = + ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections // can easily double the amount of time necessary to perform linking. diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index b34a29c25b9..3ec5c14ff17 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -6,9 +6,14 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::FnAbi; +use crate::constant::ConstantCx; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { +pub(crate) fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { let tcx = cx.tcx; let _inst_guard = @@ -18,9 +23,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mir = tcx.instance_mir(instance.def); // Declare function - let name = tcx.symbol_name(instance).name.to_string(); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap(); + let symbol_name = tcx.symbol_name(instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); + let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -39,15 +44,19 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); let mut fx = FunctionCx { cx, + module, tcx, pointer_type, + vtables: FxHashMap::default(), + constants_cx: ConstantCx::new(), instance, + symbol_name, mir, fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), @@ -55,7 +64,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In block_map, local_map: IndexVec::with_capacity(mir.local_decls.len()), caller_location: None, // set by `codegen_fn_prelude` - cold_blocks: EntitySet::new(), clif_comments, source_info_set: indexmap::IndexSet::new(), @@ -90,7 +98,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mut clif_comments = fx.clif_comments; let source_info_set = fx.source_info_set; let local_map = fx.local_map; - let cold_blocks = fx.cold_blocks; + + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); // Store function in context let context = &mut cx.cached_context; @@ -103,21 +112,15 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Perform rust specific optimizations tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function( - tcx, - instance, - context, - &cold_blocks, - &mut clif_comments, - ); + crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); }); // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. context.compute_cfg(); context.compute_domtree(); - context.eliminate_unreachable_code(cx.module.isa()).unwrap(); - context.dce(cx.module.isa()).unwrap(); + context.eliminate_unreachable_code(module.isa()).unwrap(); + context.dce(module.isa()).unwrap(); // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't // invalidate it when it would change. context.domtree.clear(); @@ -125,7 +128,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In context.want_disasm = crate::pretty_clif::should_write_ir(tcx); // Define function - let module = &mut cx.module; tcx.sess.time("define function", || { module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) @@ -136,7 +138,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In crate::pretty_clif::write_clif_file( tcx, "opt", - Some(cx.module.isa()), + Some(module.isa()), instance, &context, &clif_comments, @@ -145,13 +147,13 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - &format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(instance).name), |file| file.write_all(disasm.as_bytes()), ) } // Define debuginfo for function - let isa = cx.module.isa(); + let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; tcx.sess.time("generate debug info", || { @@ -159,7 +161,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In debug_context.define_function( instance, func_id, - &name, + symbol_name.name, isa, context, &source_info_set, @@ -205,9 +207,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { // Unwinding after panicking is not supported continue; - // FIXME once unwinding is supported uncomment next lines - // // Unwinding is unlikely to happen, so mark cleanup block's as cold. - // fx.cold_blocks.insert(block); + // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do + // so for cleanup blocks. } fx.bcx.ins().nop(); @@ -262,7 +263,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { let target = fx.get_block(*target); let failure = fx.bcx.create_block(); - fx.cold_blocks.insert(failure); + // FIXME Mark failure block as cold once Cranelift supports it if *expected { fx.bcx.ins().brz(cond, failure, &[]); @@ -355,14 +356,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { from_hir_call: _, } => { fx.tcx.sess.time("codegen call", || { - crate::abi::codegen_terminator_call( - fx, - *fn_span, - block, - func, - args, - *destination, - ) + crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination) }); } TerminatorKind::InlineAsm { @@ -664,7 +658,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -750,85 +744,15 @@ fn codegen_stmt<'tcx>( | StatementKind::AscribeUserType(..) => {} StatementKind::LlvmInlineAsm(asm) => { - use rustc_span::symbol::Symbol; - let LlvmInlineAsm { asm, outputs, inputs } = &**asm; - let rustc_hir::LlvmInlineAsmInner { - asm: asm_code, // Name - outputs: output_names, // Vec<LlvmInlineAsmOutput> - inputs: input_names, // Vec<Name> - clobbers, // Vec<Name> - volatile, // bool - alignstack, // bool - dialect: _, - asm_str_style: _, - } = asm; - match asm_code.as_str().trim() { + match asm.asm.asm.as_str().trim() { "" => { // Black box } - "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { - assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); - assert_eq!(output_names.len(), 4); - for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - assert_eq!(inputs.len(), 2); - let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax - let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx - - let (eax, ebx, ecx, edx) = - crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); - - assert_eq!(outputs.len(), 4); - codegen_place(fx, outputs[0]) - .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[1]) - .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[2]) - .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[3]) - .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); - } - "xgetbv" => { - assert_eq!(input_names, &[Symbol::intern("{ecx}")]); - - assert_eq!(output_names.len(), 2); - for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); - } - // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { - crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); - } - _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { - crate::trap::trap_unimplemented(fx, "Alloca is not supported"); - } - // Used in sys::windows::abort_internal - "int $$0x29" => { - crate::trap::trap_unimplemented(fx, "Windows abort"); - } - _ => fx - .tcx - .sess - .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), + _ => fx.tcx.sess.span_fatal( + stmt.source_info.span, + "Legacy `llvm_asm!` inline assembly is not supported. \ + Try using the new `asm!` instead.", + ), } } StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), @@ -844,7 +768,7 @@ fn codegen_stmt<'tcx>( let elem_size: u64 = pointee.size.bytes(); let bytes = if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes); } } } diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index b5874f62535..92e4435565e 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -1,8 +1,10 @@ use rustc_index::vec::IndexVec; +use rustc_middle::ty::SymbolName; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; +use crate::constant::ConstantCx; use crate::prelude::*; pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { @@ -226,12 +228,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { - pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module + pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>, + pub(crate) constants_cx: ConstantCx, pub(crate) instance: Instance<'tcx>, + pub(crate) symbol_name: SymbolName<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, pub(crate) fn_abi: Option<FnAbi<'tcx, Ty<'tcx>>>, @@ -242,9 +248,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. pub(crate) caller_location: Option<CValue<'tcx>>, - /// See [`crate::optimize::code_layout`] for more information. - pub(crate) cold_blocks: EntitySet<Block>, - pub(crate) clif_comments: crate::pretty_clif::CommentWriter, pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>, @@ -339,7 +342,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn triple(&self) -> &target_lexicon::Triple { - self.cx.module.isa().triple() + self.module.isa().triple() } pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { @@ -352,15 +355,14 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); let msg_id = self - .cx .module .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) .unwrap(); // Ignore DuplicateDefinition error, as the data will be the same - let _ = self.cx.module.define_data(msg_id, &data_ctx); + let _ = self.module.define_data(msg_id, &data_ctx); - let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs new file mode 100644 index 00000000000..e59a0cb0a23 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/config.rs @@ -0,0 +1,107 @@ +use std::env; +use std::str::FromStr; + +fn bool_env_var(key: &str) -> bool { + env::var(key).as_ref().map(|val| &**val) == Ok("1") +} + +/// The mode to use for compilation. +#[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + /// AOT compile the crate. This is the default. + Aot, + /// JIT compile and execute the crate. + Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. + JitLazy, +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. +#[derive(Clone, Debug)] +pub struct BackendConfig { + /// Should the crate be AOT compiled or JIT executed. + /// + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. + pub codegen_mode: CodegenMode, + + /// When JIT mode is enable pass these arguments to the program. + /// + /// Defaults to the value of `CG_CLIF_JIT_ARGS`. + pub jit_args: Vec<String>, + + /// Display the time it took to perform codegen for a crate. + /// + /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise. + /// Can be set using `-Cllvm-args=display_cg_time=...`. + pub display_cg_time: bool, + + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run + /// once before passing the clif ir to Cranelift for compilation. + /// + /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is + /// compiled with debug assertions enabled or false otherwise. Can be set using + /// `-Cllvm-args=enable_verifier=...`. + pub enable_verifier: bool, + + /// Don't cache object files in the incremental cache. Useful during development of cg_clif + /// to make it possible to use incremental mode for all analyses performed by rustc without + /// caching object files when their content should have been changed by a change to cg_clif. + /// + /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false + /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. + pub disable_incr_cache: bool, +} + +impl Default for BackendConfig { + fn default() -> Self { + BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: { + let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); + args.split(' ').map(|arg| arg.to_string()).collect() + }, + display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"), + enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), + disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), + } + } +} + +impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. + pub fn from_opts(opts: &[String]) -> Result<Self, String> { + fn parse_bool(name: &str, value: &str) -> Result<bool, String> { + value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) + } + + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + "display_cg_time" => config.display_cg_time = parse_bool(name, value)?, + "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, + "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + + Ok(config) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index fcd41c84465..0a0e02d2639 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -2,7 +2,7 @@ use rustc_span::DUMMY_SP; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ @@ -15,10 +15,10 @@ use cranelift_module::*; use crate::prelude::*; -#[derive(Default)] pub(crate) struct ConstantCx { todo: Vec<TodoItem>, done: FxHashSet<DataId>, + anon_allocs: FxHashMap<AllocId, DataId>, } #[derive(Copy, Clone, Debug)] @@ -28,6 +28,10 @@ enum TodoItem { } impl ConstantCx { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() } + } + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { //println!("todo {:?}", self.todo); define_all_allocs(tcx, module, &mut self); @@ -74,8 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { + let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(tcx, module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -83,8 +89,8 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("tls {:?}", def_id)); } @@ -97,8 +103,8 @@ fn codegen_static_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -182,28 +188,31 @@ pub(crate) fn codegen_const_value<'tcx>( let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); let base_addr = match alloc_kind { Some(GlobalAlloc::Memory(alloc)) => { - fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); - let data_id = - data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.module, + ptr.alloc_id, + alloc.mutability, + ); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { - let func_id = - crate::abi::import_function(fx.tcx, fx.cx.module, instance); + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); let local_func_id = - fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) } Some(GlobalAlloc::Static(def_id)) => { assert!(fx.tcx.is_static(def_id)); - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -243,10 +252,11 @@ fn pointer_for_allocation<'tcx>( alloc: &'tcx Allocation, ) -> crate::pointer::Pointer { let alloc_id = fx.tcx.create_memory_alloc(alloc); - fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); - let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", alloc_id)); } @@ -255,18 +265,14 @@ fn pointer_for_allocation<'tcx>( } fn data_id_for_alloc_id( + cx: &mut ConstantCx, module: &mut dyn Module, alloc_id: AllocId, mutability: rustc_hir::Mutability, ) -> DataId { - module - .declare_data( - &format!(".L__alloc_{:x}", alloc_id.0), - Linkage::Local, - mutability == rustc_hir::Mutability::Mut, - false, - ) - .unwrap() + *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap() + }) } fn data_id_for_static( @@ -344,7 +350,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant GlobalAlloc::Memory(alloc) => alloc, GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), }; - let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability); + let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability); (data_id, alloc, None) } TodoItem::Static(def_id) => { @@ -397,7 +403,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } GlobalAlloc::Memory(target_alloc) => { cx.todo.push(TodoItem::Alloc(reloc)); - data_id_for_alloc_id(module, reloc, target_alloc.mutability) + data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability) } GlobalAlloc::Static(def_id) => { if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) @@ -419,8 +425,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - // FIXME don't duplicate definitions in lazy jit mode - let _ = module.define_data(data_id, &data_ctx); + module.define_data(data_id, &data_ctx).unwrap(); cx.done.insert(data_id); } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs index 357c9fe6ed8..ca7083cccb8 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -5,17 +5,19 @@ use crate::prelude::*; use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; use crate::backend::WriteDebugInfo; -pub(crate) struct UnwindContext<'tcx> { - tcx: TyCtxt<'tcx>, +pub(crate) struct UnwindContext { + endian: RunTimeEndian, frame_table: FrameTable, cie_id: Option<CieId>, } -impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { +impl UnwindContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = super::target_endian(tcx); let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { @@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> { None }; - UnwindContext { tcx, frame_table, cie_id } + UnwindContext { endian, frame_table, cie_id } } pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { @@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> { } pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if !eh_frame.0.writer.slice().is_empty() { @@ -71,16 +72,12 @@ impl<'tcx> UnwindContext<'tcx> { } #[cfg(feature = "jit")] - pub(crate) unsafe fn register_jit( - self, - jit_module: &cranelift_jit::JITModule, - ) -> Option<UnwindRegistry> { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { - return None; + return; } let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); @@ -88,7 +85,10 @@ impl<'tcx> UnwindContext<'tcx> { // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); - let mut registrations = Vec::new(); + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + #[allow(unused_variables)] + let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame); // ======================================================================= // Everything after this line up to the end of the file is loosly based on @@ -96,8 +96,8 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(target_os = "macos")] { // On macOS, `__register_frame` takes a pointer to a single FDE - let start = eh_frame.as_ptr(); - let end = start.add(eh_frame.len()); + let start = eh_frame; + let end = start.add(eh_frame_len); let mut current = start; // Walk all of the entries in the frame table and register them @@ -107,7 +107,6 @@ impl<'tcx> UnwindContext<'tcx> { // Skip over the CIE if current != start { __register_frame(current); - registrations.push(current as usize); } // Move to the next table entry (+4 because the length itself is not inclusive) @@ -117,41 +116,12 @@ impl<'tcx> UnwindContext<'tcx> { #[cfg(not(target_os = "macos"))] { // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = eh_frame.as_ptr(); - __register_frame(ptr); - registrations.push(ptr as usize); + __register_frame(eh_frame); } - - Some(UnwindRegistry { _frame_table: eh_frame, registrations }) } } -/// Represents a registry of function unwind information for System V ABI. -pub(crate) struct UnwindRegistry { - _frame_table: Vec<u8>, - registrations: Vec<usize>, -} - extern "C" { // libunwind import fn __register_frame(fde: *const u8); - fn __deregister_frame(fde: *const u8); -} - -impl Drop for UnwindRegistry { - fn drop(&mut self) { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index ed3bdedddce..9c5cd53d866 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -16,12 +16,6 @@ use cranelift_object::ObjectModule; use crate::{prelude::*, BackendConfig}; -fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule { - let module = crate::backend::make_module(tcx.sess, name); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); - module -} - struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); impl<HCX> HashStable<HCX> for ModuleCodegenResult { @@ -32,11 +26,12 @@ impl<HCX> HashStable<HCX> for ModuleCodegenResult { fn emit_module( tcx: TyCtxt<'_>, + backend_config: &BackendConfig, name: String, kind: ModuleKind, module: ObjectModule, debug: Option<DebugContext<'_>>, - unwind_context: UnwindContext<'_>, + unwind_context: UnwindContext, ) -> ModuleCodegenResult { let mut product = module.finish(); @@ -52,7 +47,7 @@ fn emit_module( tcx.sess.fatal(&format!("error writing object file: {}", err)); } - let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { + let work_product = if backend_config.disable_incr_cache { None } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( @@ -110,23 +105,24 @@ fn module_codegen( let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut module = new_module(tcx, cgu_name.as_str().to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); let mut cx = crate::CodegenCx::new( tcx, - backend_config, - &mut module, + backend_config.clone(), + module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, ); - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); - } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) + cx.tcx + .sess + .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); } + MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { @@ -138,25 +134,28 @@ fn module_codegen( } } } - let (global_asm, debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); - let codegen_result = emit_module( - tcx, - cgu.name().as_str().to_string(), - ModuleKind::Regular, - module, - debug, - unwind_context, - ); + let debug_context = cx.debug_context; + let unwind_context = cx.unwind_context; + let codegen_result = tcx.sess.time("write object file", || { + emit_module( + tcx, + &backend_config, + cgu.name().as_str().to_string(), + ModuleKind::Regular, + module, + debug_context, + unwind_context, + ) + }); - codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); codegen_result } -pub(super) fn run_aot( +pub(crate) fn run_aot( tcx: TyCtxt<'_>, backend_config: BackendConfig, metadata: EncodedMetadata, @@ -193,14 +192,14 @@ pub(super) fn run_aot( } } - let modules = super::time(tcx, "codegen mono items", || { + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} + _ if backend_config.disable_incr_cache => {} CguReuse::No => {} CguReuse::PreLto => { return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); @@ -212,7 +211,7 @@ pub(super) fn run_aot( let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( dep_node, tcx, - (backend_config, cgu.name()), + (backend_config.clone(), cgu.name()), module_codegen, rustc_middle::dep_graph::hash_result, ); @@ -228,7 +227,10 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); - let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut allocator_module = + crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); + assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); @@ -236,6 +238,7 @@ pub(super) fn run_aot( let allocator_module = if created_alloc_shim { let ModuleCodegenResult(module, work_product) = emit_module( tcx, + &backend_config, "allocator_shim".to_string(), ModuleKind::Allocator, allocator_module, diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index f585ebca9dc..956207c5b7c 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -1,4 +1,4 @@ -//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. use std::cell::RefCell; @@ -8,32 +8,62 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; -thread_local! { - pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None); - pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None); +struct JitState { + backend_config: BackendConfig, + jit_module: JITModule, } -pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { - if !tcx.sess.opts.output_types.should_codegen() { - tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); - } +thread_local! { + static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None); +} +fn create_jit_module<'tcx>( + tcx: TyCtxt<'tcx>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (JITModule, CodegenCx<'tcx>) { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = - JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); - jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); + let isa = crate::build_isa(tcx.sess, backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); - assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); + + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); + + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + ); + + (jit_module, cx) +} + +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.sess.fatal("can't jit non-executable crate"); + } + + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); let mono_items = cgus @@ -44,52 +74,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::<Vec<(_, (_, _))>>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); - - super::time(tcx, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); + super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id); + crate::constant::codegen_static(tcx, &mut jit_module, def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); + let item = tcx.hir().item(item_id); tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); } } } }); - let (global_asm, _debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - jit_module.finalize_definitions(); - - if !global_asm.is_empty() { + if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); - tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; + unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" ); - let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) - .chain(args.split(' ')) + .chain(backend_config.jit_args.iter().map(|arg| &**arg)) .map(|arg| CString::new(arg).unwrap()) .collect::<Vec<_>>(); let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); @@ -98,8 +121,21 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); - BACKEND_CONFIG.with(|tls_backend_config| { - assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); @@ -161,24 +197,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 // lift is used to ensure the correct lifetime for instance. let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - CURRENT_MODULE.with(|jit_module| { - let mut jit_module = jit_module.borrow_mut(); - let jit_module = jit_module.as_mut().unwrap(); - let backend_config = - BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + let backend_config = lazy_jit_state.backend_config.clone(); - let name = tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); - let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); + let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); - let (global_asm, _debug_context, unwind_context) = cx.finalize(); - assert!(global_asm.is_empty()); + assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + unsafe { cx.unwind_context.register_jit(&jit_module) }; jit_module.get_finalized_function(func_id) }) }) @@ -248,35 +283,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { let tcx = cx.tcx; - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); - let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); - let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let name = tcx.symbol_name(inst).name; + let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); + let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); - let jit_fn = cx - .module + let jit_fn = module .declare_function( "__clif_jit_fn", Linkage::Import, &Signature { - call_conv: cx.module.target_config().default_call_conv, + call_conv: module.target_config().default_call_conv, params: vec![AbiParam::new(pointer_type)], returns: vec![AbiParam::new(pointer_type)], }, ) .unwrap(); - let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + cx.cached_context.clear(); + let trampoline = &mut cx.cached_context.func; + trampoline.signature = sig.clone(); + let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); let entry_block = trampoline_builder.create_block(); @@ -291,10 +328,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - cx.module + module .define_function( func_id, - &mut Context::for_function(trampoline), + &mut cx.cached_context, &mut NullTrapSink {}, &mut NullStackMapSink {}, ) diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index d49182a07b7..8f5714ecb41 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -1,63 +1,37 @@ -//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions -//! like JIT executing or writing object files. +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static -use std::any::Any; - -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; -use crate::CodegenMode; -mod aot; +pub(crate) mod aot; #[cfg(feature = "jit")] -mod jit; - -pub(crate) fn codegen_crate( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - backend_config: crate::BackendConfig, -) -> Box<dyn Any> { - tcx.sess.abort_if_errors(); - - match backend_config.codegen_mode { - CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - let is_executable = - tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } - - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx, backend_config); - - #[cfg(not(feature = "jit"))] - tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } - } -} +pub(crate) mod jit; fn predefine_mono_items<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { - cx.tcx.sess.time("predefine functions", || { - let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); + tcx.sess.time("predefine functions", || { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = cx.tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); - let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( mono_item, linkage, visibility, is_compiler_builtins, ); - cx.module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } @@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>( }); } -fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { - if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") { +fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R { + if display { println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name); let before = std::time::Instant::now(); let res = tcx.sess.time(name, f); diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 1fb5e86aed7..4ab4c2957ca 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -24,6 +24,64 @@ pub(crate) fn codegen_inline_asm<'tcx>( let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(1)); return; + } else if template[0] == InlineAsmTemplatePiece::String("mov rsi, rbx".to_string()) + && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[2] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[4] == InlineAsmTemplatePiece::String("xchg rsi, rbx".to_string()) + { + assert_eq!(operands.len(), 4); + let (leaf, eax_place) = match operands[0] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let ebx_place = match operands[1] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::si)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + let (sub_leaf, ecx_place) = match operands[2] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let edx_place = match operands[3] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + + let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); + + eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + return; + } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); } let mut slot_size = Size::from_bytes(0); @@ -92,8 +150,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let inline_asm_index = fx.inline_asm_index; fx.inline_asm_index += 1; - let asm_name = - format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index); + let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index); let generated_asm = generate_asm_wrapper( &asm_name, @@ -202,7 +259,6 @@ fn call_inline_asm<'tcx>( } let inline_asm_func = fx - .cx .module .declare_function( asm_name, @@ -214,7 +270,7 @@ fn call_inline_asm<'tcx>( }, ) .unwrap(); - let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(inline_asm_func, asm_name); } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs index b27b0eddfba..9de12e759bc 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs @@ -8,7 +8,7 @@ use crate::prelude::*; pub(crate) fn codegen_cpuid_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, leaf: Value, - _subleaf: Value, + _sub_leaf: Value, ) -> (Value, Value, Value, Value) { let leaf_0 = fx.bcx.create_block(); let leaf_1 = fx.bcx.create_block(); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index 83c91f789cd..ba4ed2162cd 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` - llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { + "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) { let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); @@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); }; - llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) { + "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) { let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) { 0 => FloatCC::Equal, @@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) }); }; - llvm.x86.sse2.psrli.d, (c a, o imm8) { + "llvm.x86.sse2.psrli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.pslli.d, (c a, o imm8) { + "llvm.x86.sse2.pslli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.storeu.dq, (v mem_addr, c a) { + "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) { // FIXME correctly handle the unalignment let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 39e047a98f9..435737f3a51 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -8,23 +8,25 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; +use rustc_span::symbol::{sym, kw}; +use rustc_middle::ty::print::with_no_trimmed_paths; + use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; -use rustc_middle::ty::print::with_no_trimmed_paths; macro intrinsic_pat { (_) => { _ }, ($name:ident) => { - stringify!($name) + sym::$name + }, + (kw.$name:ident) => { + kw::$name }, ($name:literal) => { - stringify!($name) + $name }, - ($x:ident . $($xs:tt).*) => { - concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) - } } macro intrinsic_arg { @@ -87,7 +89,7 @@ macro call_intrinsic_match { )*) => { match $intrinsic { $( - stringify!($name) => { + sym::$name => { assert!($substs.is_noop()); if let [$(ref $arg),*] = *$args { let ($($arg,)*) = ( @@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); let ret = match destination { Some((place, _)) => place, None => { // Insert non returning intrinsics here match intrinsic { - "abort" => { + sym::abort => { trap_abort(fx, "Called intrinsic::abort."); } - "transmute" => { + sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } _ => unimplemented!("unsupported instrinsic {}", intrinsic), @@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } }; - if intrinsic.starts_with("simd_") { + if intrinsic.as_str().starts_with("simd_") { self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); fx.bcx.ins().jump(ret_block, &[]); @@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sinf64(flt) -> f64 => sin, cosf32(flt) -> f32 => cosf, cosf64(flt) -> f64 => cos, - tanf32(flt) -> f32 => tanf, - tanf64(flt) -> f64 => tan, } intrinsic_match! { @@ -496,12 +495,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( count }; - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -515,12 +514,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::volatile_copy_nonoverlapping_memory { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; size_of_val, <T> (c ptr) { @@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); }; - _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem + | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { - "unchecked_add" => BinOp::Add, - "unchecked_sub" => BinOp::Sub, - "unchecked_div" | "exact_div" => BinOp::Div, - "unchecked_rem" => BinOp::Rem, - "unchecked_shl" => BinOp::Shl, - "unchecked_shr" => BinOp::Shr, - _ => unreachable!("intrinsic {}", intrinsic), + sym::unchecked_add => BinOp::Add, + sym::unchecked_sub => BinOp::Sub, + sym::unchecked_div | sym::exact_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::Shl, + sym::unchecked_shr => BinOp::Shr, + _ => unreachable!(), }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); }; - _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { - "add_with_overflow" => BinOp::Add, - "sub_with_overflow" => BinOp::Sub, - "mul_with_overflow" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), + sym::add_with_overflow => BinOp::Add, + sym::sub_with_overflow => BinOp::Sub, + sym::mul_with_overflow => BinOp::Mul, + _ => unreachable!(), }; let res = crate::num::codegen_checked_int_binop( @@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) { + saturating_add | saturating_sub, <T> (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { - "saturating_add" => BinOp::Add, - "saturating_sub" => BinOp::Sub, - _ => unreachable!("intrinsic {}", intrinsic), + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), }; let signed = type_sign(T); @@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); let val = match (intrinsic, signed) { - ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), - ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), - ("saturating_add", true) => { + (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), + (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } - ("saturating_sub", true) => { + (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); @@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; rotate_left, <T>(v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotl(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; rotate_right, <T>(v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotr(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; @@ -670,7 +680,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, <T> (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. @@ -806,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T), @@ -815,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T), @@ -827,7 +837,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_load let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); @@ -836,7 +845,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; volatile_store | unaligned_volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); @@ -878,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); }; - _ if intrinsic.starts_with("atomic_fence"), () { + _ if intrinsic.as_str().starts_with("atomic_fence"), () { fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_singlethreadfence"), () { + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_load"), <T> (v ptr) { + _ if intrinsic.as_str().starts_with("atomic_load"), <T> (v ptr) { validate_atomic_type!(fx, intrinsic, span, T); let ty = fx.clif_type(T).unwrap(); @@ -894,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(T)); ret.write_cvalue(fx, val); }; - _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { validate_atomic_type!(fx, intrinsic, span, val.layout().ty); let val = val.load_scalar(fx); fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); }; - _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) { + _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -913,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); @@ -927,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_val) }; - _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -939,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -951,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -963,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -975,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -989,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647 - _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1001,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1013,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1025,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1037,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1071,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); }; - try, (v f, v data, v _catch_fn) { + kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: CallConv::triple_default(fx.triple()), @@ -1088,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { let res = crate::num::codegen_float_binop(fx, match intrinsic { - "fadd_fast" => BinOp::Add, - "fsub_fast" => BinOp::Sub, - "fmul_fast" => BinOp::Mul, - "fdiv_fast" => BinOp::Div, - "frem_fast" => BinOp::Rem, + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, _ => unreachable!(), }, x, y); ret.write_cvalue(fx, res); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 27fc2abedc7..940d2514f74 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); intrinsic_match! { fx, intrinsic, substs, args, @@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U - _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { + _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); + let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 720d2a12534..32f40395702 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] @@ -23,7 +23,6 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -34,9 +33,10 @@ use rustc_middle::ty::query::Providers; use rustc_session::config::OutputFilenames; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; -use crate::constant::ConstantCx; +pub use crate::config::*; use crate::prelude::*; mod abi; @@ -49,6 +49,7 @@ mod cast; mod codegen_i128; mod common; mod compiler_builtins; +mod config; mod constant; mod debuginfo; mod discriminant; @@ -87,7 +88,6 @@ mod prelude { pub(crate) use rustc_index::vec::Idx; - pub(crate) use cranelift_codegen::entity::EntitySet; pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::types; @@ -119,95 +119,36 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> { } } -struct CodegenCx<'m, 'tcx: 'm> { +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). +struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, - module: &'m mut dyn Module, global_asm: String, - constants_cx: ConstantCx, cached_context: Context, - vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>, debug_context: Option<DebugContext<'tcx>>, - unwind_context: UnwindContext<'tcx>, + unwind_context: UnwindContext, } -impl<'m, 'tcx> CodegenCx<'m, 'tcx> { +impl<'tcx> CodegenCx<'tcx> { fn new( tcx: TyCtxt<'tcx>, backend_config: BackendConfig, - module: &'m mut dyn Module, + isa: &dyn TargetIsa, debug_info: bool, ) -> Self { - let unwind_context = UnwindContext::new( - tcx, - module.isa(), - matches!(backend_config.codegen_mode, CodegenMode::Aot), - ); - let debug_context = - if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None }; + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + + let unwind_context = + UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); + let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; CodegenCx { tcx, - module, global_asm: String::new(), - constants_cx: ConstantCx::default(), cached_context: Context::new(), - vtables: FxHashMap::default(), debug_context, unwind_context, } } - - fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) { - self.constants_cx.finalize(self.tcx, self.module); - (self.global_asm, self.debug_context, self.unwind_context) - } -} - -#[derive(Copy, Clone, Debug)] -pub enum CodegenMode { - Aot, - Jit, - JitLazy, -} - -impl Default for CodegenMode { - fn default() -> Self { - CodegenMode::Aot - } -} - -impl FromStr for CodegenMode { - type Err = String; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "aot" => Ok(CodegenMode::Aot), - "jit" => Ok(CodegenMode::Jit), - "jit-lazy" => Ok(CodegenMode::JitLazy), - _ => Err(format!("Unknown codegen mode `{}`", s)), - } - } -} - -#[derive(Copy, Clone, Debug, Default)] -pub struct BackendConfig { - pub codegen_mode: CodegenMode, -} - -impl BackendConfig { - fn from_opts(opts: &[String]) -> Result<Self, String> { - let mut config = BackendConfig::default(); - for opt in opts { - if let Some((name, value)) = opt.split_once('=') { - match name { - "mode" => config.codegen_mode = value.parse()?, - _ => return Err(format!("Unknown option `{}`", name)), - } - } else { - return Err(format!("Invalid option `{}`", opt)); - } - } - Ok(config) - } } pub struct CraneliftCodegenBackend { @@ -240,13 +181,23 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box<dyn Any> { - let config = if let Some(config) = self.config { + tcx.sess.abort_if_errors(); + let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - driver::codegen_crate(tcx, metadata, need_metadata_module, config) + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + let _: ! = driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } } fn join_codegen( @@ -284,7 +235,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> { +fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); @@ -292,9 +243,8 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> { let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided - let enable_verifier = - cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok(); - flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap(); + let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + flags_builder.set("enable_verifier", enable_verifier).unwrap(); let tls_model = match target_triple.binary_format { BinaryFormat::Elf => "elf_gd", @@ -322,10 +272,28 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> { let flags = settings::Flags::new(flags_builder); let variant = cranelift_codegen::isa::BackendVariant::MachInst; - let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); - // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` - // is interpreted as `bsr`. - isa_builder.enable("nehalem").unwrap(); + + let isa_builder = match sess.opts.cg.target_cpu.as_deref() { + Some("native") => { + let builder = cranelift_native::builder_with_options(variant, true).unwrap(); + builder + } + Some(value) => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + if let Err(_) = builder.enable(value) { + sess.fatal("The specified target cpu isn't currently supported by Cranelift."); + } + builder + } + None => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + // Don't use "haswell" as the default, as it implies `has_lzcnt`. + // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. + builder.enable("nehalem").unwrap(); + builder + } + }; + isa_builder.finish(flags) } diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs index a564a59f725..ca853aac158 100644 --- a/compiler/rustc_codegen_cranelift/src/linkage.rs +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage( (RLinkage::External, Visibility::Default) => Linkage::Export, (RLinkage::Internal, Visibility::Default) => Linkage::Local, (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible, _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), } } diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index dc86290b3fe..d504024a335 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -1,6 +1,9 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_hir::LangItem; +use rustc_middle::ty::subst::GenericArg; +use rustc_middle::ty::AssocKind; use rustc_session::config::EntryFnType; +use rustc_span::symbol::Ident; use crate::prelude::*; @@ -9,9 +12,10 @@ use crate::prelude::*; pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, + is_jit: bool, ) { - let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( def_id, match entry_ty { @@ -23,18 +27,19 @@ pub(crate) fn maybe_create_entry_wrapper( }; let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, rust_main_def_id: DefId, - use_start_lang_item: bool, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -57,9 +62,9 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let main_name = tcx.symbol_name(instance).name.to_string(); + let main_name = tcx.symbol_name(instance).name; let main_sig = get_function_sig(tcx, m.isa().triple(), instance); - let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap(); + let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); let mut ctx = Context::new(); ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig); @@ -74,7 +79,47 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); - let call_inst = if use_start_lang_item { + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + let call_inst = bcx.ins().call(main_func_ref, &[]); + let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); + + let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let report = tcx + .associated_items(termination_trait) + .find_by_name_and_kind( + tcx, + Ident::from_str("report"), + AssocKind::Fn, + termination_trait, + ) + .unwrap(); + let report = Instance::resolve( + tcx, + ParamEnv::reveal_all(), + report.def_id, + tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()), + ) + .unwrap() + .unwrap(); + + let report_name = tcx.symbol_name(report).name; + let report_sig = get_function_sig(tcx, m.isa().triple(), report); + let report_func_id = + m.declare_function(report_name, Linkage::Import, &report_sig).unwrap(); + let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func); + + // FIXME do proper abi handling instead of expecting the pass mode to be identical + // for returns and arguments. + let report_call_inst = bcx.ins().call(report_func_ref, &call_results); + let res = bcx.func.dfg.inst_results(report_call_inst)[0]; + match m.target_config().pointer_type() { + types::I32 => res, + types::I64 => bcx.ins().sextend(types::I64, res), + _ => unimplemented!("16bit systems are not yet supported"), + } + } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( tcx, @@ -90,13 +135,14 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] } else { // using user-defined start fn - bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] }; - let result = bcx.inst_results(call_inst)[0]; bcx.ins().return_(&[result]); bcx.seal_all_blocks(); bcx.finalize(); diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs index dbdc8cbad44..882232fde09 100644 --- a/compiler/rustc_codegen_cranelift/src/metadata.rs +++ b/compiler/rustc_codegen_cranelift/src/metadata.rs @@ -8,13 +8,24 @@ use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; +use rustc_middle::middle::cstore::MetadataLoader; use rustc_middle::ty::TyCtxt; -use rustc_session::config; use rustc_target::spec::Target; use crate::backend::WriteMetadata; +/// The metadata loader used by cg_clif. +/// +/// The metadata is stored in the same format as cg_llvm. +/// +/// # Metadata location +/// +/// <dl> +/// <dt>rlib</dt> +/// <dd>The metadata can be found in the `lib.rmeta` file inside of the ar archive.</dd> +/// <dt>dylib</dt> +/// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd> +/// </dl> pub(crate) struct CraneliftMetadataLoader; fn load_metadata_with( @@ -58,54 +69,16 @@ impl MetadataLoader for CraneliftMetadataLoader { } // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 -pub(crate) fn write_metadata<P: WriteMetadata>( - tcx: TyCtxt<'_>, - product: &mut P, -) -> EncodedMetadata { +pub(crate) fn write_metadata<O: WriteMetadata>(tcx: TyCtxt<'_>, object: &mut O) { use snap::write::FrameEncoder; use std::io::Write; - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - config::CrateType::Executable - | config::CrateType::Staticlib - | config::CrateType::Cdylib => MetadataKind::None, - - config::CrateType::Rlib => MetadataKind::Uncompressed, - - config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - if kind == MetadataKind::None { - return EncodedMetadata::new(); - } - let metadata = tcx.encode_metadata(); - if kind == MetadataKind::Uncompressed { - return metadata; - } - - assert!(kind == MetadataKind::Compressed); let mut compressed = tcx.metadata_encoding_version(); FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); - product.add_rustc_section( + object.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), compressed, - tcx.sess.target.is_like_osx, ); - - metadata } diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index 2ebf30da2d8..b6d378a5fe1 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let val_hi = fx.bcx.ins().umulhi(lhs, rhs); fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) } else { + // Based on LLVM's instruction sequence for compiling + // a.checked_mul(b).is_some() to riscv64gc: + // mulh a2, a0, a1 + // mul a0, a0, a1 + // srai a0, a0, 63 + // xor a0, a0, a2 + // snez a0, a0 let val_hi = fx.bcx.ins().smulhi(lhs, rhs); - let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0); - let not_all_ones = fx.bcx.ins().icmp_imm( - IntCC::NotEqual, - val_hi, - u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64, - ); - fx.bcx.ins().band(not_all_zero, not_all_ones) + let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let xor = fx.bcx.ins().bxor(val_hi, val_sign); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) }; (val, has_overflow) } diff --git a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs deleted file mode 100644 index ca9ff15ec10..00000000000 --- a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! This optimization moves cold code to the end of the function. -//! -//! Some code is executed much less often than other code. For example panicking or the -//! landingpads for unwinding. By moving this cold code to the end of the function the average -//! amount of jumps is reduced and the code locality is improved. -//! -//! # Undefined behaviour -//! -//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself. - -use crate::prelude::*; - -pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block>) { - // FIXME Move the block in place instead of remove and append once - // bytecodealliance/cranelift#1339 is implemented. - - let mut block_insts = FxHashMap::default(); - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>(); - for &inst in &insts { - ctx.func.layout.remove_inst(inst); - } - block_insts.insert(block, insts); - ctx.func.layout.remove_block(block); - } - - // And then append them at the back again. - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - ctx.func.layout.append_block(block); - for inst in block_insts.remove(&block).unwrap() { - ctx.func.layout.append_inst(inst, block); - } - } -} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs index 389f50e797e..137fb5f7731 100644 --- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs @@ -2,29 +2,16 @@ use crate::prelude::*; -mod code_layout; pub(crate) mod peephole; -mod stack2reg; pub(crate) fn optimize_function<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ctx: &mut Context, - cold_blocks: &EntitySet<Block>, clif_comments: &mut crate::pretty_clif::CommentWriter, ) { - // The code_layout optimization is very cheap. - self::code_layout::optimize_function(ctx, cold_blocks); + // FIXME classify optimizations over opt levels once we have more - if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { - return; // FIXME classify optimizations over opt levels - } - - // FIXME(#1142) stack2reg miscompiles lewton - if false { - self::stack2reg::optimize_function(ctx, clif_comments); - } - - crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments); + crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments); crate::base::verify_func(tcx, &*clif_comments, &ctx.func); } diff --git a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs deleted file mode 100644 index 8bb02a3e558..00000000000 --- a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs +++ /dev/null @@ -1,486 +0,0 @@ -//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible. -//! -//! # Undefined behaviour -//! -//! This optimization is based on the assumption that stack slots which don't have their address -//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the -//! function which has the stack slots. This optimization also assumes that stack slot accesses -//! are never out of bounds. If these assumptions are not correct, then this optimization may remove -//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value -//! being loaded by a `stack_load`. - -use std::collections::BTreeMap; -use std::fmt; -use std::ops::Not; - -use rustc_data_structures::fx::FxHashSet; - -use cranelift_codegen::cursor::{Cursor, FuncCursor}; -use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; - -use crate::prelude::*; - -/// Workaround for `StackSlot` not implementing `Ord`. -#[derive(Copy, Clone, PartialEq, Eq)] -struct OrdStackSlot(StackSlot); - -impl fmt::Debug for OrdStackSlot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl PartialOrd for OrdStackSlot { - fn partial_cmp(&self, rhs: &Self) -> Option<std::cmp::Ordering> { - self.0.as_u32().partial_cmp(&rhs.0.as_u32()) - } -} - -impl Ord for OrdStackSlot { - fn cmp(&self, rhs: &Self) -> std::cmp::Ordering { - self.0.as_u32().cmp(&rhs.0.as_u32()) - } -} - -#[derive(Debug, Default)] -struct StackSlotUsage { - stack_addr: FxHashSet<Inst>, - stack_load: FxHashSet<Inst>, - stack_store: FxHashSet<Inst>, -} - -impl StackSlotUsage { - fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec<Inst> { - self.stack_store - .iter() - .cloned() - .filter(|&store| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&store| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::<Vec<Inst>>() - } - - fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec<Inst> { - self.stack_load - .iter() - .cloned() - .filter(|&load| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&load| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::<Vec<Inst>>() - } - - fn remove_unused_stack_addr(func: &mut Function, inst: Inst) { - func.dfg.detach_results(inst); - func.dfg.replace(inst).nop(); - } - - fn remove_unused_load(func: &mut Function, load: Inst) { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - } - - fn remove_dead_store(&mut self, func: &mut Function, store: Inst) { - func.dfg.replace(store).nop(); - self.stack_store.remove(&store); - } - - fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) { - let loaded_value = func.dfg.inst_results(load)[0]; - let loaded_type = func.dfg.value_type(loaded_value); - - if func.dfg.value_type(value) == loaded_type { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - func.dfg.change_to_alias(loaded_value, value); - } else { - func.dfg.replace(load).bitcast(loaded_type, value); - } - - self.stack_load.remove(&load); - } -} - -struct OptimizeContext<'a> { - ctx: &'a mut Context, - stack_slot_usage_map: BTreeMap<OrdStackSlot, StackSlotUsage>, -} - -impl<'a> OptimizeContext<'a> { - fn for_context(ctx: &'a mut Context) -> Self { - ctx.flowgraph(); // Compute cfg and domtree. - - // Record all stack_addr, stack_load and stack_store instructions. - let mut stack_slot_usage_map = BTreeMap::<OrdStackSlot, StackSlotUsage>::new(); - - let mut cursor = FuncCursor::new(&mut ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::StackLoad { - opcode: Opcode::StackAddr, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_addr - .insert(inst); - } - InstructionData::StackLoad { - opcode: Opcode::StackLoad, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_load - .insert(inst); - } - InstructionData::StackStore { - opcode: Opcode::StackStore, - arg: _, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_store - .insert(inst); - } - _ => {} - } - } - } - - OptimizeContext { ctx, stack_slot_usage_map } - } -} - -pub(super) fn optimize_function( - ctx: &mut Context, - clif_comments: &mut crate::pretty_clif::CommentWriter, -) { - combine_stack_addr_with_load_store(&mut ctx.func); - - let mut opt_ctx = OptimizeContext::for_context(ctx); - - // FIXME Repeat following instructions until fixpoint. - - remove_unused_stack_addr_and_stack_load(&mut opt_ctx); - - if clif_comments.enabled() { - for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { - clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); - } - } - - for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() { - if users.stack_addr.is_empty().not() { - // Stack addr leaked; there may be unknown loads and stores. - // FIXME use stacked borrows to optimize - continue; - } - - for load in users.stack_load.clone().into_iter() { - let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); - - if clif_comments.enabled() { - for &store in &potential_stores { - clif_comments.add_comment( - load, - format!( - "Potential store -> load forwarding {} -> {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(store, None), - opt_ctx.ctx.func.dfg.display_inst(load, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - match *potential_stores { - [] => { - if clif_comments.enabled() { - clif_comments - .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); - } - } - [store] - if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full - && temporal_order(&opt_ctx.ctx, store, load) - == TemporalOrder::DefinitivelyBefore => - { - // Only one store could have been the origin of the value. - let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; - - if clif_comments.enabled() { - clif_comments.add_comment( - load, - format!("Store to load forward {} -> {}", store, load), - ); - } - - users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); - } - _ => {} // FIXME implement this - } - } - - for store in users.stack_store.clone().into_iter() { - let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); - - if clif_comments.enabled() { - for &load in &potential_loads { - clif_comments.add_comment( - store, - format!( - "Potential load from store {} <- {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(load, None), - opt_ctx.ctx.func.dfg.display_inst(store, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - if potential_loads.is_empty() { - // Never loaded; can safely remove all stores and the stack slot. - // FIXME also remove stores when there is always a next store before a load. - - if clif_comments.enabled() { - clif_comments.add_comment( - store, - format!( - "Remove dead stack store {} of {}", - opt_ctx.ctx.func.dfg.display_inst(store, None), - stack_slot.0 - ), - ); - } - - users.remove_dead_store(&mut opt_ctx.ctx.func, store); - } - } - - if users.stack_store.is_empty() && users.stack_load.is_empty() { - opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0; - } - } -} - -fn combine_stack_addr_with_load_store(func: &mut Function) { - // Turn load and store into stack_load and stack_store when possible. - let mut cursor = FuncCursor::new(func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUD: stack_load.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - let ty = cursor.func.dfg.ctrl_typevar(inst); - cursor.func.dfg.replace(inst).stack_load( - ty, - stack_slot, - combined_offset, - ); - } - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [value, addr], - flags: _, - offset, - } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUND: stack_store.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - cursor.func.dfg.replace(inst).stack_store( - value, - stack_slot, - combined_offset, - ); - } - } - } - _ => {} - } - } - } -} - -fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) { - // FIXME incrementally rebuild on each call? - let mut stack_addr_load_insts_users = FxHashMap::<Inst, FxHashSet<Inst>>::default(); - - let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - for &arg in cursor.func.dfg.inst_args(inst) { - if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) { - match cursor.func.dfg[arg_origin].opcode() { - Opcode::StackAddr | Opcode::StackLoad => { - stack_addr_load_insts_users - .entry(arg_origin) - .or_insert_with(FxHashSet::default) - .insert(inst); - } - _ => {} - } - } - } - } - } - - #[cfg(debug_assertions)] - for inst in stack_addr_load_insts_users.keys() { - let mut is_recorded_stack_addr_or_stack_load = false; - for stack_slot_users in opt_ctx.stack_slot_usage_map.values() { - is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst) - || stack_slot_users.stack_load.contains(inst); - } - assert!(is_recorded_stack_addr_or_stack_load); - } - - // Replace all unused stack_addr and stack_load instructions with nop. - let mut func = &mut opt_ctx.ctx.func; - - for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() { - stack_slot_users - .stack_addr - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst)); - - stack_slot_users - .stack_load - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst)); - } -} - -fn try_get_stack_slot_and_offset_for_addr( - func: &Function, - addr: Value, -) -> Option<(StackSlot, Offset32)> { - if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) { - if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } = - func.dfg[addr_inst] - { - return Some((stack_slot, offset)); - } - } - None -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum SpatialOverlap { - No, - Partial, - Full, -} - -fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap { - fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) { - match func.dfg[inst] { - InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } - | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset } - | InstructionData::StackStore { - opcode: Opcode::StackStore, - stack_slot, - offset, - arg: _, - } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()), - _ => unreachable!("{:?}", func.dfg[inst]), - } - } - - debug_assert_ne!(src, dest); - - let (src_ss, src_offset, src_size) = inst_info(func, src); - let (dest_ss, dest_offset, dest_size) = inst_info(func, dest); - - if src_ss != dest_ss { - return SpatialOverlap::No; - } - - if src_offset == dest_offset && src_size == dest_size { - return SpatialOverlap::Full; - } - - let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into(); - let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into(); - if src_end <= dest_offset.into() || dest_end <= src_offset.into() { - return SpatialOverlap::No; - } - - SpatialOverlap::Partial -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum TemporalOrder { - /// `src` will never be executed before `dest`. - NeverBefore, - - /// `src` may be executed before `dest`. - MaybeBefore, - - /// `src` will always be executed before `dest`. - /// There may still be other instructions in between. - DefinitivelyBefore, -} - -fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder { - debug_assert_ne!(src, dest); - - if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::DefinitivelyBefore - } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::NeverBefore - } else { - TemporalOrder::MaybeBefore - } -} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index d22ea3772ee..158811c5eaf 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -207,7 +207,7 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { pub(crate) fn write_ir_file( tcx: TyCtxt<'_>, - name: &str, + name: impl FnOnce() -> String, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { if !should_write_ir(tcx) { @@ -222,7 +222,7 @@ pub(crate) fn write_ir_file( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(name); + let clif_file_name = clif_output_dir.join(name()); let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { @@ -238,30 +238,31 @@ pub(crate) fn write_clif_file<'tcx>( context: &cranelift_codegen::Context, mut clif_comments: &CommentWriter, ) { - write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| { - let value_ranges = - isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); + write_ir_file( + tcx, + || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa + .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa(tcx.sess)), - value_ranges: value_ranges.as_ref(), - }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, + ) + .unwrap(); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); } impl fmt::Debug for FunctionCx<'_, '_, '_> { diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs index 1ab0703e981..819c8b51558 100644 --- a/compiler/rustc_codegen_cranelift/src/trap.rs +++ b/compiler/rustc_codegen_cranelift/src/trap.rs @@ -4,7 +4,6 @@ use crate::prelude::*; fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { let puts = fx - .cx .module .declare_function( "puts", @@ -16,13 +15,12 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { }, ) .unwrap(); - let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } - let symbol_name = fx.tcx.symbol_name(fx.instance); - let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg); + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); let msg_ptr = fx.anonymous_str("trap", &real_msg); fx.bcx.ins().call(puts, &[msg_ptr]); } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index b97d3900984..9a572c3501f 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -554,7 +554,7 @@ impl<'tcx> CPlace<'tcx> { let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memory_copy( - fx.cx.module.target_config(), + fx.module.target_config(), to_addr, from_addr, size, diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index 9053d1aa1b0..bbf07ffc85d 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -72,15 +72,15 @@ pub(crate) fn get_vtable<'tcx>( layout: TyAndLayout<'tcx>, trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, ) -> Value { - let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) { + let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) { *data_id } else { let data_id = build_vtable(fx, layout, trait_ref); - fx.cx.vtables.insert((layout.ty, trait_ref), data_id); + fx.vtables.insert((layout.ty, trait_ref), data_id); data_id }; - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } @@ -94,7 +94,7 @@ fn build_vtable<'tcx>( let drop_in_place_fn = import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), ); @@ -111,7 +111,7 @@ fn build_vtable<'tcx>( opt_mth.map(|(def_id, substs)| { import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) .unwrap() .polymorphize(fx.tcx), @@ -132,34 +132,16 @@ fn build_vtable<'tcx>( for (i, component) in components.into_iter().enumerate() { if let Some(func_id) = component { - let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr((i * usize_size) as u32, func_ref); } } data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx - .cx - .module - .declare_data( - &format!( - "__vtable.{}.for.{:?}.{}", - trait_ref - .as_ref() - .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into()) - .unwrap_or(std::borrow::Cow::Borrowed("???")), - layout.ty, - fx.cx.vtables.len(), - ), - Linkage::Local, - false, - false, - ) - .unwrap(); - - // FIXME don't duplicate definitions in lazy jit mode - let _ = fx.cx.module.define_data(data_id, &data_ctx); + let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); + + fx.module.define_data(data_id, &data_ctx).unwrap(); data_id } |