From 27e6ee102ed359a3744ac1f2dffbfb7a60bd2371 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 26 Dec 2023 04:20:35 +0000 Subject: Add callbr support to LLVM wrapper --- compiler/rustc_codegen_llvm/src/builder.rs | 52 +++++++++++++++++++++++++++++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 14 ++++++++ 2 files changed, 66 insertions(+) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 8cab2a3f27c..eda3c583994 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1547,6 +1547,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } + pub(crate) fn callbr( + &mut self, + llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + default_dest: &'ll BasicBlock, + indirect_dest: &[&'ll BasicBlock], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("callbr", llty, llfn, args); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } + + let callbr = unsafe { + llvm::LLVMRustBuildCallBr( + self.llbuilder, + llty, + llfn, + default_dest, + indirect_dest.as_ptr(), + indirect_dest.len() as c_uint, + args.as_ptr(), + args.len() as c_uint, + bundles.as_ptr(), + bundles.len() as c_uint, + UNNAMED, + ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, callbr); + } + callbr + } + // Emits CFI pointer type membership tests. fn cfi_type_test( &mut self, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index dbf35e5f499..22c67436d36 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1617,6 +1617,20 @@ extern "C" { Name: *const c_char, ) -> &'a Value; + pub fn LLVMRustBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); pub fn LLVMRustSetAlgebraicMath(Instr: &Value); pub fn LLVMRustSetAllowReassoc(Instr: &Value); -- cgit 1.4.1-3-g733a5 From 5e4fd6bc2301a8e267d044a526ffb77c1a6c35d7 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 27 Dec 2023 04:08:04 +0000 Subject: Implement asm goto for LLVM and GCC backend --- compiler/rustc_codegen_gcc/src/asm.rs | 31 ++++++++++++++++++++--- compiler/rustc_codegen_llvm/src/asm.rs | 29 +++++++++++++-------- compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 ++ compiler/rustc_codegen_ssa/src/mir/block.rs | 38 ++++++++++++++++++++++------ compiler/rustc_codegen_ssa/src/traits/asm.rs | 6 ++++- 5 files changed, 84 insertions(+), 22 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index 78e8e32b972..07edd26e27a 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -107,7 +107,7 @@ enum ConstraintOrRegister { impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { - fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) { + fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) { if options.contains(InlineAsmOptions::MAY_UNWIND) { self.sess().dcx() .create_err(UnwindingInlineAsm { span: span[0] }) @@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // added to `outputs.len()` let mut inputs = vec![]; + // GCC index of a label equals its position in the array added to + // `outputs.len() + inputs.len()`. + let mut labels = vec![]; + // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _` let mut clobbers = vec![]; @@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // some targets to add a leading underscore (Mach-O). constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); } + + InlineAsmOperandRef::Label { label } => { + labels.push(label); + } } } @@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { .. } => { // processed in the previous pass } + + InlineAsmOperandRef::Label { .. } => { + // processed in the previous pass + } } } @@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { ref string } => { template_str.push_str(string); } + + InlineAsmOperandRef::Label { label } => { + let label_gcc_index = labels.iter() + .position(|&l| l == label) + .expect("wrong rust index"); + let gcc_index = label_gcc_index + outputs.len() + inputs.len(); + push_to_template(Some('l'), gcc_index); + } } } } @@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // 4. Generate Extended Asm block let block = self.llbb(); - let extended_asm = block.add_extended_asm(None, &template_str); + let extended_asm = if let Some(dest) = dest { + assert!(!labels.is_empty()); + block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest)) + } else { + block.add_extended_asm(None, &template_str) + }; for op in &outputs { extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var); @@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { if !options.contains(InlineAsmOptions::NOSTACK) { // TODO(@Commeownist): figure out how to align stack } - if options.contains(InlineAsmOptions::NORETURN) { + if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) { let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable"); let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) }; self.call(self.type_void(), None, None, builtin_unreachable, &[], None); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a413466093b..f5d0b3c4657 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Build the template string + let mut labels = vec![]; let mut template_str = String::new(); for piece in template { match *piece { @@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Only emit the raw symbol name template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } + InlineAsmOperandRef::Label { label } => { + template_str.push_str(&format!("${{{}:l}}", constraints.len())); + constraints.push("!i".to_owned()); + labels.push(label); + } } } } @@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &constraints.join(","), &inputs, output_type, + &labels, volatile, alignstack, dialect, line_spans, options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, + dest, + catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { + if let Some(dest) = dest { self.switch_to_block(dest); } @@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>( cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, + labels: &[&'ll llvm::BasicBlock], volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, + dest: Option<&'ll llvm::BasicBlock>, + catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; @@ -457,8 +464,10 @@ pub(crate) fn inline_asm_call<'ll>( can_throw, ); - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) + let call = if !labels.is_empty() { + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) } else { bx.call(fty, None, None, v, inputs, None) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 23e6f054a7c..ee76038800f 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -377,12 +377,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { constraint, inputs, self.type_void(), + &[], true, false, llvm::AsmDialect::Att, &[span], false, None, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index c9a1e7789c2..2eba489cbfb 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -264,7 +264,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { mir::UnwindAction::Unreachable => None, }; - if let Some(cleanup) = unwind_target { + if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) { let ret_llbb = if let Some(target) = destination { fx.llbb(target) } else { @@ -277,11 +277,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { options, line_spans, instance, - Some((ret_llbb, cleanup, self.funclet(fx))), + Some(ret_llbb), + None, + ); + MergingSucc::False + } else if let Some(cleanup) = unwind_target { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + operands, + options, + line_spans, + instance, + Some(ret_llbb), + Some((cleanup, self.funclet(fx))), ); MergingSucc::False } else { - bx.codegen_inline_asm(template, operands, options, line_spans, instance, None); + bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None); if let Some(target) = destination { self.funclet_br(fx, bx, target, mergeable_succ) @@ -1067,7 +1085,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands: &[mir::InlineAsmOperand<'tcx>], options: ast::InlineAsmOptions, line_spans: &[Span], - destination: Option, + targets: &[mir::BasicBlock], unwind: mir::UnwindAction, instance: Instance<'_>, mergeable_succ: bool, @@ -1119,8 +1137,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::InlineAsmOperand::SymStatic { def_id } => { InlineAsmOperandRef::SymStatic { def_id } } - mir::InlineAsmOperand::Label { target_index: _ } => { - todo!(); + mir::InlineAsmOperand::Label { target_index } => { + InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) } } }) .collect(); @@ -1132,7 +1150,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &operands, options, line_spans, - destination, + if options.contains(InlineAsmOptions::NORETURN) { + None + } else { + targets.get(0).copied() + }, unwind, instance, mergeable_succ, @@ -1298,7 +1320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands, options, line_spans, - targets.get(0).copied(), + targets, unwind, self.instance, mergeable_succ(), diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index c2ae74b18d8..8d67b626bbd 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { SymStatic { def_id: DefId, }, + Label { + label: B::BasicBlock, + }, } #[derive(Debug)] @@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ); } -- cgit 1.4.1-3-g733a5 From 626a5f589208d544537f411724f35e6b841790e4 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 24 Feb 2024 17:12:25 +0000 Subject: Add assertions and clarify asm-goto with noreturn --- compiler/rustc_codegen_llvm/src/asm.rs | 1 + compiler/rustc_codegen_ssa/src/mir/block.rs | 1 + src/doc/unstable-book/src/language-features/asm-goto.md | 8 +++++++- 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index f5d0b3c4657..74539d4d495 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -465,6 +465,7 @@ pub(crate) fn inline_asm_call<'ll>( ); let call = if !labels.is_empty() { + assert!(catch_funclet.is_none()); bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) } else if let Some((catch, funclet)) = catch_funclet { bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 2eba489cbfb..8a2f91c082e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -265,6 +265,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { }; if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) { + assert!(unwind_target.is_none()); let ret_llbb = if let Some(target) = destination { fx.llbb(target) } else { diff --git a/src/doc/unstable-book/src/language-features/asm-goto.md b/src/doc/unstable-book/src/language-features/asm-goto.md index a9ad5e83744..d72eb7c0c6e 100644 --- a/src/doc/unstable-book/src/language-features/asm-goto.md +++ b/src/doc/unstable-book/src/language-features/asm-goto.md @@ -21,4 +21,10 @@ unsafe { } ``` -The block must have unit type. +The block must have unit type or diverge. + +When `label ` is used together with `noreturn` option, it means that the +assembly will not fallthrough. It's allowed to jump to a label within the +assembly. In this case, the entire `asm!` expression will have an unit type as +opposed to diverging, if not all label blocks diverge. The `asm!` expression +still diverges if `noreturn` option is used and all label blocks diverge. -- cgit 1.4.1-3-g733a5