diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/abi/mod.rs | 46 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/base.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/codegen_i128.rs | 27 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/config.rs | 80 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/driver/aot.rs | 276 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/driver/jit.rs | 29 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/inline_asm.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/lib.rs | 59 |
9 files changed, 263 insertions, 291 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index cab5b35c18d..1c706694dcd 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -125,8 +125,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { returns: Vec<AbiParam>, args: &[Value], ) -> Cow<'_, [Value]> { - if self.tcx.sess.target.is_like_windows { - let (mut params, mut args): (Vec<_>, Vec<_>) = params + // Pass i128 arguments by-ref on Windows. + let (params, args): (Vec<_>, Cow<'_, [_]>) = if self.tcx.sess.target.is_like_windows { + let (params, args): (Vec<_>, Vec<_>) = params .into_iter() .zip(args) .map(|(param, &arg)| { @@ -140,29 +141,42 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { }) .unzip(); - let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128; + (params, args.into()) + } else { + (params, args.into()) + }; - if indirect_ret_val { - params.insert(0, AbiParam::new(self.pointer_type)); - let ret_ptr = self.create_stack_slot(16, 16); - args.insert(0, ret_ptr.get_addr(self)); - self.lib_call_unadjusted(name, params, vec![], &args); - return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]); + // Return i128 using a return area pointer on Windows and s390x. + let adjust_ret_param = + if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" { + returns.len() == 1 && returns[0].value_type == types::I128 } else { - return self.lib_call_unadjusted(name, params, returns, &args); - } - } + false + }; + + if adjust_ret_param { + let mut params = params; + let mut args = args.to_vec(); - self.lib_call_unadjusted(name, params, returns, args) + params.insert(0, AbiParam::new(self.pointer_type)); + let ret_ptr = self.create_stack_slot(16, 16); + args.insert(0, ret_ptr.get_addr(self)); + + self.lib_call_unadjusted(name, params, vec![], &args); + + Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]) + } else { + Cow::Borrowed(self.lib_call_unadjusted(name, params, returns, &args)) + } } - pub(crate) fn lib_call_unadjusted( + fn lib_call_unadjusted( &mut self, name: &str, params: Vec<AbiParam>, returns: Vec<AbiParam>, args: &[Value], - ) -> Cow<'_, [Value]> { + ) -> &[Value] { let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv }; let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); @@ -175,7 +189,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } let results = self.bcx.inst_results(call_inst); assert!(results.len() <= 2, "{}", results.len()); - Cow::Borrowed(results) + results } } diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 06cc5754894..85cf4c7361a 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -6,6 +6,7 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_module::ModuleError; use rustc_ast::InlineAsmOptions; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_index::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::InlineAsmMacro; @@ -16,6 +17,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; +use crate::enable_verifier; use crate::inline_asm::codegen_naked_asm; use crate::prelude::*; use crate::pretty_clif::CommentWriter; @@ -169,12 +171,13 @@ pub(crate) fn codegen_fn<'tcx>( pub(crate) fn compile_fn( cx: &mut crate::CodegenCx, + profiler: &SelfProfilerRef, cached_context: &mut Context, module: &mut dyn Module, codegened_func: CodegenedFunction, ) { let _timer = - cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); + profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); let clif_comments = codegened_func.clif_comments; @@ -212,7 +215,7 @@ pub(crate) fn compile_fn( }; // Define function - cx.profiler.generic_activity("define function").run(|| { + profiler.generic_activity("define function").run(|| { context.want_disasm = cx.should_write_ir; match module.define_function(codegened_func.func_id, context) { Ok(()) => {} @@ -253,7 +256,7 @@ pub(crate) fn compile_fn( // Define debuginfo for function let debug_context = &mut cx.debug_context; - cx.profiler.generic_activity("generate debug info").run(|| { + profiler.generic_activity("generate debug info").run(|| { if let Some(debug_context) = debug_context { codegened_func.func_debug_cx.unwrap().finalize( debug_context, @@ -264,11 +267,11 @@ pub(crate) fn compile_fn( }); } -pub(crate) fn verify_func( - tcx: TyCtxt<'_>, - writer: &crate::pretty_clif::CommentWriter, - func: &Function, -) { +fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func: &Function) { + if !enable_verifier(tcx.sess) { + return; + } + tcx.prof.generic_activity("verify clif ir").run(|| { let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); match cranelift_codegen::verify_function(&func, &flags) { diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs index b6a4769e031..734574338d0 100644 --- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -81,26 +81,6 @@ pub(crate) fn maybe_codegen_checked<'tcx>( match bin_op { BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => unreachable!(), BinOp::Add | BinOp::Sub => None, - BinOp::Mul if is_signed => { - let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]); - let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32)); - let lhs = lhs.load_scalar(fx); - let rhs = rhs.load_scalar(fx); - let oflow_ptr = oflow.to_ptr().get_addr(fx); - let res = fx.lib_call_unadjusted( - "__muloti4", - vec![ - AbiParam::new(types::I128), - AbiParam::new(types::I128), - AbiParam::new(fx.pointer_type), - ], - vec![AbiParam::new(types::I128)], - &[lhs, rhs, oflow_ptr], - )[0]; - let oflow = oflow.to_cvalue(fx).load_scalar(fx); - let oflow = fx.bcx.ins().ireduce(types::I8, oflow); - Some(CValue::by_val_pair(res, oflow, fx.layout_of(out_ty))) - } BinOp::Mul => { let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]); let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); @@ -110,7 +90,12 @@ pub(crate) fn maybe_codegen_checked<'tcx>( AbiParam::new(types::I128), ]; let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; - fx.lib_call("__rust_u128_mulo", param_types, vec![], &args); + fx.lib_call( + if is_signed { "__rust_i128_mulo" } else { "__rust_u128_mulo" }, + param_types, + vec![], + &args, + ); Some(out_place.to_cvalue(fx)) } BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(), diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs index 12bce680d9e..d784f6e9d9e 100644 --- a/compiler/rustc_codegen_cranelift/src/config.rs +++ b/compiler/rustc_codegen_cranelift/src/config.rs @@ -1,10 +1,3 @@ -use std::env; -use std::str::FromStr; - -fn bool_env_var(key: &str) -> bool { - env::var(key).as_deref() == Ok("1") -} - /// The mode to use for compilation. #[derive(Copy, Clone, Debug)] pub enum CodegenMode { @@ -16,19 +9,6 @@ pub enum CodegenMode { JitLazy, } -impl FromStr for CodegenMode { - type Err = String; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "aot" => Ok(CodegenMode::Aot), - "jit" => Ok(CodegenMode::Jit), - "jit-lazy" => Ok(CodegenMode::JitLazy), - _ => Err(format!("Unknown codegen mode `{}`", s)), - } - } -} - /// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. #[derive(Clone, Debug)] pub struct BackendConfig { @@ -41,51 +21,22 @@ pub struct BackendConfig { /// /// Defaults to the value of `CG_CLIF_JIT_ARGS`. pub jit_args: Vec<String>, - - /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run - /// once before passing the clif ir to Cranelift for compilation. - /// - /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is - /// compiled with debug assertions enabled or false otherwise. Can be set using - /// `-Cllvm-args=enable_verifier=...`. - pub enable_verifier: bool, - - /// Don't cache object files in the incremental cache. Useful during development of cg_clif - /// to make it possible to use incremental mode for all analyses performed by rustc without - /// caching object files when their content should have been changed by a change to cg_clif. - /// - /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false - /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. - pub disable_incr_cache: bool, -} - -impl Default for BackendConfig { - fn default() -> Self { - BackendConfig { - codegen_mode: CodegenMode::Aot, - jit_args: { - match std::env::var("CG_CLIF_JIT_ARGS") { - Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(), - Err(std::env::VarError::NotPresent) => vec![], - Err(std::env::VarError::NotUnicode(s)) => { - panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s); - } - } - }, - enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), - disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), - } - } } impl BackendConfig { /// Parse the configuration passed in using `-Cllvm-args`. pub fn from_opts(opts: &[String]) -> Result<Self, String> { - fn parse_bool(name: &str, value: &str) -> Result<bool, String> { - value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) - } + let mut config = BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: match std::env::var("CG_CLIF_JIT_ARGS") { + Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(), + Err(std::env::VarError::NotPresent) => vec![], + Err(std::env::VarError::NotUnicode(s)) => { + panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s); + } + }, + }; - let mut config = BackendConfig::default(); for opt in opts { if opt.starts_with("-import-instr-limit") { // Silently ignore -import-instr-limit. It is set by rust's build system even when @@ -94,9 +45,14 @@ impl BackendConfig { } if let Some((name, value)) = opt.split_once('=') { match name { - "mode" => config.codegen_mode = value.parse()?, - "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, - "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, + "mode" => { + config.codegen_mode = match value { + "aot" => CodegenMode::Aot, + "jit" => CodegenMode::Jit, + "jit-lazy" => CodegenMode::JitLazy, + _ => return Err(format!("Unknown codegen mode `{}`", value)), + }; + } _ => return Err(format!("Unknown option `{}`", name)), } } else { diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 8eab73ad5f9..5bbcfc2cda7 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -1,6 +1,7 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::env; use std::fs::{self, File}; use std::io::BufWriter; use std::path::{Path, PathBuf}; @@ -25,13 +26,18 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::Session; use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType}; -use crate::BackendConfig; +use crate::CodegenCx; +use crate::base::CodegenedFunction; use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; use crate::debuginfo::TypeDebugContext; use crate::global_asm::GlobalAsmConfig; use crate::prelude::*; use crate::unwind_module::UnwindModule; +fn disable_incr_cache() -> bool { + env::var("CG_CLIF_DISABLE_INCR_CACHE").as_deref() == Ok("1") +} + struct ModuleCodegenResult { module_regular: CompiledModule, module_global_asm: Option<CompiledModule>, @@ -63,10 +69,10 @@ impl OngoingCodegen { self, sess: &Session, outputs: &OutputFilenames, - backend_config: &BackendConfig, ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let mut work_products = FxIndexMap::default(); let mut modules = vec![]; + let disable_incr_cache = disable_incr_cache(); for module_codegen in self.modules { let module_codegen_result = match module_codegen { @@ -87,7 +93,7 @@ impl OngoingCodegen { if let Some((work_product_id, work_product)) = existing_work_product { work_products.insert(work_product_id, work_product); } else { - let work_product = if backend_config.disable_incr_cache { + let work_product = if disable_incr_cache { None } else if let Some(module_global_asm) = &module_global_asm { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( @@ -322,12 +328,8 @@ fn produce_final_output_artifacts( // These are used in linking steps and will be cleaned up afterward. } -fn make_module( - sess: &Session, - backend_config: &BackendConfig, - name: String, -) -> UnwindModule<ObjectModule> { - let isa = crate::build_isa(sess, backend_config); +fn make_module(sess: &Session, name: String) -> UnwindModule<ObjectModule> { + let isa = crate::build_isa(sess); let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); @@ -412,7 +414,13 @@ fn emit_module( Err(err) => return Err(format!("error writing object file: {}", err)), }; - prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); + if prof.enabled() { + prof.artifact_size( + "object_file", + tmp_file.file_name().unwrap().to_string_lossy(), + file.metadata().unwrap().len(), + ); + } Ok(CompiledModule { name, @@ -486,91 +494,101 @@ fn reuse_workproduct_for_cgu( }) } +fn codegen_cgu_content( + tcx: TyCtxt<'_>, + module: &mut dyn Module, + cgu_name: rustc_span::Symbol, +) -> (CodegenCx, Vec<CodegenedFunction>) { + let _timer = tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()); + + let cgu = tcx.codegen_unit(cgu_name); + let mono_items = cgu.items_in_deterministic_order(tcx); + + let mut cx = crate::CodegenCx::new( + tcx, + module.isa(), + tcx.sess.opts.debuginfo != DebugInfo::None, + cgu_name, + ); + let mut type_dbg = TypeDebugContext::default(); + super::predefine_mono_items(tcx, module, &mono_items); + let mut codegened_functions = vec![]; + for (mono_item, _) in mono_items { + match mono_item { + MonoItem::Fn(inst) => { + if let Some(codegened_function) = crate::base::codegen_fn( + tcx, + &mut cx, + &mut type_dbg, + Function::new(), + module, + inst, + ) { + codegened_functions.push(codegened_function); + } + } + MonoItem::Static(def_id) => { + let data_id = crate::constant::codegen_static(tcx, module, def_id); + if let Some(debug_context) = &mut cx.debug_context { + debug_context.define_static(tcx, &mut type_dbg, def_id, data_id); + } + } + MonoItem::GlobalAsm(item_id) => { + crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id); + } + } + } + crate::main_shim::maybe_create_entry_wrapper(tcx, module, false, cgu.is_primary()); + + (cx, codegened_functions) +} + fn module_codegen( tcx: TyCtxt<'_>, - (backend_config, global_asm_config, cgu_name, token): ( - BackendConfig, + (global_asm_config, cgu_name, token): ( Arc<GlobalAsmConfig>, rustc_span::Symbol, ConcurrencyLimiterToken, ), ) -> OngoingModuleCodegen { - let (cgu_name, mut cx, mut module, codegened_functions) = - tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| { - let cgu = tcx.codegen_unit(cgu_name); - let mono_items = cgu.items_in_deterministic_order(tcx); - - let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string()); - - let mut cx = crate::CodegenCx::new( - tcx, - module.isa(), - tcx.sess.opts.debuginfo != DebugInfo::None, - cgu_name, - ); - let mut type_dbg = TypeDebugContext::default(); - super::predefine_mono_items(tcx, &mut module, &mono_items); - let mut codegened_functions = vec![]; - for (mono_item, _) in mono_items { - match mono_item { - MonoItem::Fn(inst) => { - if let Some(codegened_function) = crate::base::codegen_fn( - tcx, - &mut cx, - &mut type_dbg, - Function::new(), - &mut module, - inst, - ) { - codegened_functions.push(codegened_function); - } - } - MonoItem::Static(def_id) => { - let data_id = crate::constant::codegen_static(tcx, &mut module, def_id); - if let Some(debug_context) = &mut cx.debug_context { - debug_context.define_static(tcx, &mut type_dbg, def_id, data_id); - } - } - MonoItem::GlobalAsm(item_id) => { - crate::global_asm::codegen_global_asm_item( - tcx, - &mut cx.global_asm, - item_id, - ); - } - } - } - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, false, cgu.is_primary()); + let mut module = make_module(tcx.sess, cgu_name.as_str().to_string()); - let cgu_name = cgu.name().as_str().to_owned(); + let (mut cx, codegened_functions) = codegen_cgu_content(tcx, &mut module, cgu_name); - (cgu_name, cx, module, codegened_functions) - }); + let cgu_name = cgu_name.as_str().to_owned(); let producer = crate::debuginfo::producer(tcx.sess); + let profiler = tcx.prof.clone(); + OngoingModuleCodegen::Async(std::thread::spawn(move || { - cx.profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { + profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( - cx.profiler.clone(), + profiler.clone(), ))); let mut cached_context = Context::new(); for codegened_func in codegened_functions { - crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func); + crate::base::compile_fn( + &mut cx, + &profiler, + &mut cached_context, + &mut module, + codegened_func, + ); } }); let global_asm_object_file = - cx.profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { + profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm) })?; let codegen_result = - cx.profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { + profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { emit_cgu( &global_asm_config.output_filenames, - &cx.profiler, + &profiler, cgu_name, module, cx.debug_context, @@ -583,9 +601,63 @@ fn module_codegen( })) } +fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> CompiledModule { + use rustc_middle::mir::mono::CodegenUnitNameBuilder; + + let _timer = tcx.sess.timer("write compressed metadata"); + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); + let metadata_cgu_name = cgu_name_builder + .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata")) + .as_str() + .to_string(); + + let tmp_file = + tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + + let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx); + let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name); + + if let Err(err) = std::fs::write(&tmp_file, obj) { + tcx.dcx().fatal(format!("error writing metadata object file: {}", err)); + } + + CompiledModule { + name: metadata_cgu_name, + kind: ModuleKind::Metadata, + object: Some(tmp_file), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + } +} + +fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> { + let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string()); + let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); + + if created_alloc_shim { + let product = allocator_module.finish(); + + match emit_module( + tcx.output_filenames(()), + &tcx.sess.prof, + product.object, + ModuleKind::Allocator, + "allocator_shim".to_owned(), + &crate::debuginfo::producer(tcx.sess), + ) { + Ok(allocator_module) => Some(allocator_module), + Err(err) => tcx.dcx().fatal(err), + } + } else { + None + } +} + pub(crate) fn run_aot( tcx: TyCtxt<'_>, - backend_config: BackendConfig, metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box<OngoingCodegen> { @@ -631,9 +703,10 @@ pub(crate) fn run_aot( let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + let disable_incr_cache = disable_incr_cache(); let (todo_cgus, done_cgus) = cgus.into_iter().enumerate().partition::<Vec<_>, _>(|&(i, _)| match cgu_reuse[i] { - _ if backend_config.disable_incr_cache => true, + _ if disable_incr_cache => true, CguReuse::No => true, CguReuse::PreLto | CguReuse::PostLto => false, }); @@ -647,12 +720,7 @@ pub(crate) fn run_aot( .with_task( dep_node, tcx, - ( - backend_config.clone(), - global_asm_config.clone(), - cgu.name(), - concurrency_limiter.acquire(tcx.dcx()), - ), + (global_asm_config.clone(), cgu.name(), concurrency_limiter.acquire(tcx.dcx())), module_codegen, Some(rustc_middle::dep_graph::hash_result), ) @@ -666,62 +734,10 @@ pub(crate) fn run_aot( modules }); - let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); - let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); - - let allocator_module = if created_alloc_shim { - let product = allocator_module.finish(); + let allocator_module = emit_allocator_module(tcx); - match emit_module( - tcx.output_filenames(()), - &tcx.sess.prof, - product.object, - ModuleKind::Allocator, - "allocator_shim".to_owned(), - &crate::debuginfo::producer(tcx.sess), - ) { - Ok(allocator_module) => Some(allocator_module), - Err(err) => tcx.dcx().fatal(err), - } - } else { - None - }; - - let metadata_module = if need_metadata_module { - let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || { - use rustc_middle::mir::mono::CodegenUnitNameBuilder; - - let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); - let metadata_cgu_name = cgu_name_builder - .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata")) - .as_str() - .to_string(); - - let tmp_file = - tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); - - let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx); - let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name); - - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.dcx().fatal(format!("error writing metadata object file: {}", err)); - } - - (metadata_cgu_name, tmp_file) - }); - - Some(CompiledModule { - name: metadata_cgu_name, - kind: ModuleKind::Metadata, - object: Some(tmp_file), - dwarf_object: None, - bytecode: None, - assembly: None, - llvm_ir: None, - }) - } else { - None - }; + let metadata_module = + if need_metadata_module { Some(emit_metadata_module(tcx, &metadata)) } else { None }; Box::new(OngoingCodegen { modules, diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index ae9578eeffb..d68948966ea 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -11,12 +11,12 @@ use cranelift_jit::{JITBuilder, JITModule}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; use rustc_session::Session; -use rustc_span::Symbol; +use rustc_span::sym; use crate::debuginfo::TypeDebugContext; use crate::prelude::*; use crate::unwind_module::UnwindModule; -use crate::{BackendConfig, CodegenCx, CodegenMode}; +use crate::{CodegenCx, CodegenMode}; struct JitState { jit_module: UnwindModule<JITModule>, @@ -59,14 +59,10 @@ impl UnsafeMessage { } } -fn create_jit_module( - tcx: TyCtxt<'_>, - backend_config: &BackendConfig, - hotswap: bool, -) -> (UnwindModule<JITModule>, CodegenCx) { +fn create_jit_module(tcx: TyCtxt<'_>, hotswap: bool) -> (UnwindModule<JITModule>, CodegenCx) { let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); - let isa = crate::build_isa(tcx.sess, backend_config); + let isa = crate::build_isa(tcx.sess); let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); jit_builder.hotswap(hotswap); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); @@ -81,7 +77,7 @@ fn create_jit_module( (jit_module, cx) } -pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { +pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec<String>) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.dcx().fatal("JIT mode doesn't work with `cargo check`"); } @@ -90,11 +86,8 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { tcx.dcx().fatal("can't jit non-executable crate"); } - let (mut jit_module, mut cx) = create_jit_module( - tcx, - &backend_config, - matches!(backend_config.codegen_mode, CodegenMode::JitLazy), - ); + let (mut jit_module, mut cx) = + create_jit_module(tcx, matches!(codegen_mode, CodegenMode::JitLazy)); let mut cached_context = Context::new(); let (_, cgus) = tcx.collect_and_partition_mono_items(()); @@ -110,7 +103,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { - MonoItem::Fn(inst) => match backend_config.codegen_mode { + MonoItem::Fn(inst) => match codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { codegen_and_compile_fn( @@ -151,7 +144,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { ); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) - .chain(backend_config.jit_args.iter().map(|arg| &**arg)) + .chain(jit_args.iter().map(|arg| &**arg)) .map(|arg| CString::new(arg).unwrap()) .collect::<Vec<_>>(); @@ -211,7 +204,7 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( instance: Instance<'tcx>, ) { cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( - cx.profiler.clone(), + tcx.prof.clone(), ))); tcx.prof.generic_activity("codegen and compile fn").run(|| { @@ -227,7 +220,7 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( module, instance, ) { - crate::base::compile_fn(cx, cached_context, module, codegened_func); + crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func); } }); } diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index d74c366a87f..33726056cc1 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -102,13 +102,12 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( // Pass a wrapper rather than the function itself as the function itself may not // be exported from the main codegen unit and may thus be unreachable from the // object file created by an external assembler. - let inline_asm_index = fx.cx.inline_asm_index.get(); - fx.cx.inline_asm_index.set(inline_asm_index + 1); let wrapper_name = format!( "__inline_asm_{}_wrapper_n{}", fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), - inline_asm_index + fx.cx.inline_asm_index ); + fx.cx.inline_asm_index += 1; let sig = get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance); create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name); @@ -167,13 +166,12 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>( asm_gen.allocate_registers(); asm_gen.allocate_stack_slots(); - let inline_asm_index = fx.cx.inline_asm_index.get(); - fx.cx.inline_asm_index.set(inline_asm_index + 1); let asm_name = format!( "__inline_asm_{}_n{}", fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), - inline_asm_index + fx.cx.inline_asm_index ); + fx.cx.inline_asm_index += 1; let generated_asm = asm_gen.generate_asm_wrapper(&asm_name); fx.cx.global_asm.push_str(&generated_asm); @@ -266,13 +264,12 @@ pub(crate) fn codegen_naked_asm<'tcx>( // Pass a wrapper rather than the function itself as the function itself may not // be exported from the main codegen unit and may thus be unreachable from the // object file created by an external assembler. - let inline_asm_index = cx.inline_asm_index.get(); - cx.inline_asm_index.set(inline_asm_index + 1); let wrapper_name = format!( "__inline_asm_{}_wrapper_n{}", cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), - inline_asm_index + cx.inline_asm_index ); + cx.inline_asm_index += 1; let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); create_wrapper_function(module, sig, &wrapper_name, symbol.name); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 3318c0797ec..5f1b71eff6b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -1270,8 +1270,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } sym::cold_path => { - // This is a no-op. The intrinsic is just a hint to the optimizer. - // We still have an impl here to avoid it being turned into a call. + fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); } // Unimplemented intrinsics must have a fallback body. The fallback body is obtained diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index cac9975f04c..c9486a730e1 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -34,7 +34,7 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::cell::{Cell, RefCell}; +use std::env; use std::sync::Arc; use cranelift_codegen::isa::TargetIsa; @@ -42,7 +42,6 @@ use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::CodegenResults; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::traits::CodegenBackend; -use rustc_data_structures::profiling::SelfProfilerRef; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -123,11 +122,10 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> { /// The codegen context holds any information shared between the codegen of individual functions /// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). struct CodegenCx { - profiler: SelfProfilerRef, output_filenames: Arc<OutputFilenames>, should_write_ir: bool, global_asm: String, - inline_asm_index: Cell<usize>, + inline_asm_index: usize, debug_context: Option<DebugContext>, cgu_name: Symbol, } @@ -142,11 +140,10 @@ impl CodegenCx { None }; CodegenCx { - profiler: tcx.prof.clone(), output_filenames: tcx.output_filenames(()).clone(), should_write_ir: crate::pretty_clif::should_write_ir(tcx), global_asm: String::new(), - inline_asm_index: Cell::new(0), + inline_asm_index: 0, debug_context, cgu_name, } @@ -154,7 +151,7 @@ impl CodegenCx { } pub struct CraneliftCodegenBackend { - pub config: RefCell<Option<BackendConfig>>, + pub config: Option<BackendConfig>, } impl CodegenBackend for CraneliftCodegenBackend { @@ -176,13 +173,6 @@ impl CodegenBackend for CraneliftCodegenBackend { sess.dcx() .fatal("`-Cinstrument-coverage` is LLVM specific and not supported by Cranelift"); } - - let mut config = self.config.borrow_mut(); - if config.is_none() { - let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) - .unwrap_or_else(|err| sess.dcx().fatal(err)); - *config = Some(new_config); - } } fn target_features(&self, sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> { @@ -215,12 +205,15 @@ impl CodegenBackend for CraneliftCodegenBackend { need_metadata_module: bool, ) -> Box<dyn Any> { tcx.dcx().abort_if_errors(); - let config = self.config.borrow().clone().unwrap(); + let config = self.config.clone().unwrap_or_else(|| { + BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) + .unwrap_or_else(|err| tcx.sess.dcx().fatal(err)) + }); match config.codegen_mode { - CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Aot => driver::aot::run_aot(tcx, metadata, need_metadata_module), CodegenMode::Jit | CodegenMode::JitLazy => { #[cfg(feature = "jit")] - driver::jit::run_jit(tcx, config); + driver::jit::run_jit(tcx, config.codegen_mode, config.jit_args); #[cfg(not(feature = "jit"))] tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); @@ -236,14 +229,20 @@ impl CodegenBackend for CraneliftCodegenBackend { ) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) { let _timer = sess.timer("finish_ongoing_codegen"); - ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join( - sess, - outputs, - self.config.borrow().as_ref().unwrap(), - ) + ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join(sess, outputs) } } +/// Determine if the Cranelift ir verifier should run. +/// +/// Returns true when `-Zverify-llvm-ir` is passed, the `CG_CLIF_ENABLE_VERIFIER` env var is set to +/// 1 or when cg_clif is compiled with debug assertions enabled or false otherwise. +fn enable_verifier(sess: &Session) -> bool { + sess.verify_llvm_ir() + || cfg!(debug_assertions) + || env::var("CG_CLIF_ENABLE_VERIFIER").as_deref() == Ok("1") +} + fn target_triple(sess: &Session) -> target_lexicon::Triple { // FIXME(madsmtm): Use `sess.target.llvm_target` once target-lexicon supports unversioned macOS. // See <https://github.com/bytecodealliance/target-lexicon/pull/113> @@ -253,14 +252,14 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { } } -fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIsa + 'static> { +fn build_isa(sess: &Session) -> Arc<dyn TargetIsa + 'static> { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); - let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + let enable_verifier = if enable_verifier(sess) { "true" } else { "false" }; flags_builder.set("enable_verifier", enable_verifier).unwrap(); flags_builder.set("regalloc_checker", enable_verifier).unwrap(); @@ -295,6 +294,16 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIs } } + if let target_lexicon::OperatingSystem::Windows = target_triple.operating_system { + // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510 + flags_builder.enable("enable_multi_ret_implicit_sret").unwrap(); + } + + if let target_lexicon::Architecture::S390x = target_triple.architecture { + // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510 + flags_builder.enable("enable_multi_ret_implicit_sret").unwrap(); + } + if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::Riscv64(_) | target_lexicon::Architecture::X86_64 = target_triple.architecture @@ -347,5 +356,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIs /// This is the entrypoint for a hot plugged rustc_codegen_cranelift #[no_mangle] pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> { - Box::new(CraneliftCodegenBackend { config: RefCell::new(None) }) + Box::new(CraneliftCodegenBackend { config: None }) } |