diff options
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/base.rs | 21 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 29 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/analyze.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/block.rs | 62 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/traits/asm.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 8 |
7 files changed, 109 insertions, 18 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 1b30edd2938..5cf6d95412b 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -239,7 +239,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { fx.add_comment(inst, terminator_head); } - fx.set_debug_loc(bb_data.terminator().source_info); + let source_info = bb_data.terminator().source_info; + fx.set_debug_loc(source_info); match &bb_data.terminator().kind { TerminatorKind::Goto { target } => { @@ -295,19 +296,19 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { let len = codegen_operand(fx, len).load_scalar(fx); let index = codegen_operand(fx, index).load_scalar(fx); let location = fx - .get_caller_location(bb_data.terminator().source_info.span) + .get_caller_location(source_info.span) .load_scalar(fx); codegen_panic_inner( fx, rustc_hir::LangItem::PanicBoundsCheck, &[index, len, location], - bb_data.terminator().source_info.span, + source_info.span, ); } _ => { let msg_str = msg.description(); - codegen_panic(fx, msg_str, bb_data.terminator().source_info.span); + codegen_panic(fx, msg_str, source_info.span); } } } @@ -378,10 +379,18 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { options, destination, line_spans: _, + cleanup, } => { + if cleanup.is_some() { + fx.tcx.sess.span_fatal( + source_info.span, + "cranelift doesn't support unwinding from inline assembly.", + ); + } + crate::inline_asm::codegen_inline_asm( fx, - bb_data.terminator().source_info.span, + source_info.span, template, operands, *options, @@ -415,7 +424,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { } TerminatorKind::Drop { place, target, unwind: _ } => { let drop_place = codegen_place(fx, *place); - crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place); + crate::abi::codegen_drop(fx, source_info.span, drop_place); let target_block = fx.get_block(*target); fx.bcx.ins().jump(target_block, &[]); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index aa979c6d77c..83c5cb6f1cf 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -1,6 +1,8 @@ use crate::builder::Builder; +use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -98,6 +100,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { ia.alignstack, ia.dialect, &[span], + false, + None, ); if r.is_none() { return false; @@ -121,6 +125,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -355,6 +360,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { alignstack, dialect, line_spans, + options.contains(InlineAsmOptions::MAY_UNWIND), + dest_catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -447,10 +454,16 @@ pub(crate) fn inline_asm_call( alignstack: bool, dia: LlvmAsmDialect, line_spans: &[Span], + unwind: bool, + dest_catch_funclet: Option<( + &'ll llvm::BasicBlock, + &'ll llvm::BasicBlock, + Option<&Funclet<'ll>>, + )>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; - let can_throw = llvm::False; + let can_throw = if unwind { llvm::True } else { llvm::False }; let argtys = inputs .iter() @@ -467,6 +480,13 @@ pub(crate) fn inline_asm_call( let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { + if unwind && llvm_util::get_version() < (13, 0, 0) { + bx.cx.sess().span_fatal( + line_spans[0], + "unwinding from inline assembly is only supported on llvm >= 13.", + ); + } + let v = llvm::LLVMRustInlineAsm( fty, asm.as_ptr().cast(), @@ -478,7 +498,12 @@ pub(crate) fn inline_asm_call( llvm::AsmDialect::from_generic(dia), can_throw, ); - let call = bx.call(fty, v, inputs, None); + + let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { + bx.invoke(fty, v, inputs, dest, catch, funclet) + } else { + bx.call(fty, v, inputs, None) + }; // Store mark in a metadata node so we can map LLVM errors // back to source locations. See #17552. diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index a7e34b08059..5e7d7552daf 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -350,6 +350,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { false, ast::LlvmAsmDialect::Att, &[span], + false, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index c486c19f9c9..0447c02fdec 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -276,9 +276,9 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi | TerminatorKind::SwitchInt { .. } | TerminatorKind::Yield { .. } | TerminatorKind::FalseEdge { .. } - | TerminatorKind::FalseUnwind { .. } - | TerminatorKind::InlineAsm { .. } => { /* nothing to do */ } + | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ } TerminatorKind::Call { cleanup: unwind, .. } + | TerminatorKind::InlineAsm { cleanup: unwind, .. } | TerminatorKind::Assert { cleanup: unwind, .. } | TerminatorKind::DropAndReplace { unwind, .. } | TerminatorKind::Drop { unwind, .. } => { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 495fe13f18c..e914e493269 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -10,6 +10,7 @@ use crate::traits::*; use crate::MemFlags; use rustc_ast as ast; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_hir::lang_items::LangItem; use rustc_index::vec::Idx; use rustc_middle::mir::AssertKind; @@ -174,6 +175,45 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } } } + + /// Generates inline assembly with optional `destination` and `cleanup`. + fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>( + &self, + fx: &mut FunctionCx<'a, 'tcx, Bx>, + bx: &mut Bx, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Bx>], + options: InlineAsmOptions, + line_spans: &[Span], + destination: Option<mir::BasicBlock>, + cleanup: Option<mir::BasicBlock>, + instance: Instance<'_>, + ) { + if let Some(cleanup) = cleanup { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + &operands, + options, + line_spans, + instance, + Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))), + ); + } else { + bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None); + + if let Some(target) = destination { + self.funclet_br(fx, bx, target); + } else { + bx.unreachable(); + } + } + } } /// Codegen implementations for some terminator variants. @@ -877,6 +917,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options: ast::InlineAsmOptions, line_spans: &[Span], destination: Option<mir::BasicBlock>, + cleanup: Option<mir::BasicBlock>, instance: Instance<'_>, ) { let span = terminator.source_info.span; @@ -931,13 +972,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }) .collect(); - bx.codegen_inline_asm(template, &operands, options, line_spans, instance); - - if let Some(target) = destination { - helper.funclet_br(self, &mut bx, target); - } else { - bx.unreachable(); - } + helper.do_inlineasm( + self, + &mut bx, + template, + &operands, + options, + line_spans, + destination, + cleanup, + instance, + ); } } @@ -1041,7 +1086,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options, line_spans, destination, - cleanup: _, // TODO + cleanup, } => { self.codegen_asm_terminator( helper, @@ -1052,6 +1097,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { options, line_spans, destination, + cleanup, self.instance, ); } diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index 31f539e1b03..65f3c754d2d 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -59,6 +59,7 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, ); } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index ccfc6bdc735..3fbf020c552 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -446,11 +446,19 @@ LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, char *Constraints, size_t ConstraintsLen, LLVMBool HasSideEffects, LLVMBool IsAlignStack, LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { +#if LLVM_VERSION_GE(13, 0) return wrap(InlineAsm::get(unwrap<FunctionType>(Ty), StringRef(AsmString, AsmStringLen), StringRef(Constraints, ConstraintsLen), HasSideEffects, IsAlignStack, fromRust(Dialect), CanThrow)); +#else + return wrap(InlineAsm::get(unwrap<FunctionType>(Ty), + StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), + HasSideEffects, IsAlignStack, + fromRust(Dialect))); +#endif } extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, |