diff options
| author | bors <bors@rust-lang.org> | 2024-03-08 09:34:05 +0000 |
|---|---|---|
| committer | bors <bors@rust-lang.org> | 2024-03-08 09:34:05 +0000 |
| commit | 1b2c53a15dba7962cfc284c3b6d61a0341ffa27a (patch) | |
| tree | b30523abff74295295f275d9497b5ab8e899e37e /compiler | |
| parent | 14fbc3c00525b41a3a3ee2c90e9ab6fd3b05274f (diff) | |
| parent | 0d235ef9a49a703eba2c6ee9693c30222ac5466b (diff) | |
| download | rust-1b2c53a15dba7962cfc284c3b6d61a0341ffa27a.tar.gz rust-1b2c53a15dba7962cfc284c3b6d61a0341ffa27a.zip | |
Auto merge of #122182 - matthiaskrgr:rollup-gzimi4c, r=matthiaskrgr
Rollup of 8 pull requests Successful merges: - #118623 (Improve std::fs::read_to_string example) - #119365 (Add asm goto support to `asm!`) - #120608 (Docs for std::ptr::slice_from_raw_parts) - #121832 (Add new Tier-3 target: `loongarch64-unknown-linux-musl`) - #121938 (Fix quadratic behavior of repeated vectored writes) - #122099 (Add `#[inline]` to `BTreeMap::new` constructor) - #122103 (Make TAITs and ATPITs capture late-bound lifetimes in scope) - #122143 (PassWrapper: update for llvm/llvm-project@a3319371970b) Failed merges: - #122076 (Tweak the way we protect in-place function arguments in interpreters) r? `@ghost` `@rustbot` modify labels: rollup
Diffstat (limited to 'compiler')
79 files changed, 671 insertions, 317 deletions
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 13b1a589d9b..d522c285e3e 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -2302,6 +2302,9 @@ pub enum InlineAsmOperand { Sym { sym: InlineAsmSym, }, + Label { + block: P<Block>, + }, } impl InlineAsmOperand { @@ -2311,7 +2314,7 @@ impl InlineAsmOperand { | Self::Out { reg, .. } | Self::InOut { reg, .. } | Self::SplitInOut { reg, .. } => Some(reg), - Self::Const { .. } | Self::Sym { .. } => None, + Self::Const { .. } | Self::Sym { .. } | Self::Label { .. } => None, } } } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index c517c6138ce..9ec92c9d4ed 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -1331,6 +1331,7 @@ pub fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) { } InlineAsmOperand::Const { anon_const } => vis.visit_anon_const(anon_const), InlineAsmOperand::Sym { sym } => vis.visit_inline_asm_sym(sym), + InlineAsmOperand::Label { block } => vis.visit_block(block), } } } diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index 9efb87e53cd..7296e29301f 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -823,6 +823,7 @@ pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm) try_visit!(visitor.visit_anon_const(anon_const)) } InlineAsmOperand::Sym { sym } => try_visit!(visitor.visit_inline_asm_sym(sym)), + InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)), } } V::Result::output() diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl index e87cf05713c..d91d65497e1 100644 --- a/compiler/rustc_ast_lowering/messages.ftl +++ b/compiler/rustc_ast_lowering/messages.ftl @@ -88,6 +88,9 @@ ast_lowering_invalid_abi_suggestion = did you mean ast_lowering_invalid_asm_template_modifier_const = asm template modifiers are not allowed for `const` arguments +ast_lowering_invalid_asm_template_modifier_label = + asm template modifiers are not allowed for `label` arguments + ast_lowering_invalid_asm_template_modifier_reg_class = invalid asm template modifier for this register class diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs index 7fd419f62e4..cef50a70534 100644 --- a/compiler/rustc_ast_lowering/src/asm.rs +++ b/compiler/rustc_ast_lowering/src/asm.rs @@ -3,9 +3,9 @@ use crate::{ImplTraitContext, ImplTraitPosition, ParamMode, ResolverAstLoweringE use super::errors::{ AbiSpecifiedMultipleTimes, AttSyntaxOnlyX86, ClobberAbiNotSupported, InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst, - InvalidAsmTemplateModifierRegClass, InvalidAsmTemplateModifierRegClassSub, - InvalidAsmTemplateModifierSym, InvalidRegister, InvalidRegisterClass, RegisterClassOnlyClobber, - RegisterConflict, + InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass, + InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister, + InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict, }; use super::LoweringContext; @@ -237,6 +237,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } } } + InlineAsmOperand::Label { block } => { + if !self.tcx.features().asm_goto { + feature_err( + sess, + sym::asm_goto, + *op_sp, + "label operands for inline assembly are unstable", + ) + .emit(); + } + hir::InlineAsmOperand::Label { block: self.lower_block(block, false) } + } }; (op, self.lower_span(*op_sp)) }) @@ -296,6 +308,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { op_span: op_sp, }); } + hir::InlineAsmOperand::Label { .. } => { + self.dcx().emit_err(InvalidAsmTemplateModifierLabel { + placeholder_span, + op_span: op_sp, + }); + } } } } @@ -335,7 +353,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } - | hir::InlineAsmOperand::SymStatic { .. } => { + | hir::InlineAsmOperand::SymStatic { .. } + | hir::InlineAsmOperand::Label { .. } => { unreachable!("{op:?} is not a register operand"); } }; diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs index 570449513bf..76744ae6264 100644 --- a/compiler/rustc_ast_lowering/src/errors.rs +++ b/compiler/rustc_ast_lowering/src/errors.rs @@ -262,6 +262,16 @@ pub struct InvalidAsmTemplateModifierSym { } #[derive(Diagnostic, Clone, Copy)] +#[diag(ast_lowering_invalid_asm_template_modifier_label)] +pub struct InvalidAsmTemplateModifierLabel { + #[primary_span] + #[label(ast_lowering_template_modifier)] + pub placeholder_span: Span, + #[label(ast_lowering_argument)] + pub op_span: Span, +} + +#[derive(Diagnostic, Clone, Copy)] #[diag(ast_lowering_register_class_only_clobber)] pub struct RegisterClassOnlyClobber { #[primary_span] diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index 01b1e6fcaff..52300138186 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -275,7 +275,13 @@ impl<'hir> LoweringContext<'_, 'hir> { } Some(ty) => this.lower_ty( ty, - ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: false }, + ImplTraitContext::OpaqueTy { + origin: hir::OpaqueTyOrigin::TyAlias { + parent: this.local_def_id(id), + in_assoc_ty: false, + }, + fn_kind: None, + }, ), }, ); @@ -936,7 +942,13 @@ impl<'hir> LoweringContext<'_, 'hir> { Some(ty) => { let ty = this.lower_ty( ty, - ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: true }, + ImplTraitContext::OpaqueTy { + origin: hir::OpaqueTyOrigin::TyAlias { + parent: this.local_def_id(i.id), + in_assoc_ty: true, + }, + fn_kind: None, + }, ); hir::ImplItemKind::Type(ty) } diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 03ad75ca8b4..94e1e06a954 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -265,13 +265,12 @@ enum ImplTraitContext { /// Example: `fn foo() -> impl Debug`, where `impl Debug` is conceptually /// equivalent to a new opaque type like `type T = impl Debug; fn foo() -> T`. /// - ReturnPositionOpaqueTy { - /// Origin: Either OpaqueTyOrigin::FnReturn or OpaqueTyOrigin::AsyncFn, + OpaqueTy { origin: hir::OpaqueTyOrigin, - fn_kind: FnDeclKind, + /// Only used to change the lifetime capture rules, since + /// RPITIT captures all in scope, RPIT does not. + fn_kind: Option<FnDeclKind>, }, - /// Impl trait in type aliases. - TypeAliasesOpaqueTy { in_assoc_ty: bool }, /// `impl Trait` is unstably accepted in this position. FeatureGated(ImplTraitPosition, Symbol), /// `impl Trait` is not accepted in this position. @@ -1075,9 +1074,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // Disallow ATB in dyn types if self.is_in_dyn_type { let suggestion = match itctx { - ImplTraitContext::ReturnPositionOpaqueTy { .. } - | ImplTraitContext::TypeAliasesOpaqueTy { .. } - | ImplTraitContext::Universal => { + ImplTraitContext::OpaqueTy { .. } | ImplTraitContext::Universal => { let bound_end_span = constraint .gen_args .as_ref() @@ -1417,24 +1414,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { TyKind::ImplTrait(def_node_id, bounds) => { let span = t.span; match itctx { - ImplTraitContext::ReturnPositionOpaqueTy { origin, fn_kind } => self - .lower_opaque_impl_trait( - span, - origin, - *def_node_id, - bounds, - Some(fn_kind), - itctx, - ), - ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty } => self - .lower_opaque_impl_trait( - span, - hir::OpaqueTyOrigin::TyAlias { in_assoc_ty }, - *def_node_id, - bounds, - None, - itctx, - ), + ImplTraitContext::OpaqueTy { origin, fn_kind } => self.lower_opaque_impl_trait( + span, + origin, + *def_node_id, + bounds, + fn_kind, + itctx, + ), ImplTraitContext::Universal => { let span = t.span; @@ -1553,9 +1540,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let captured_lifetimes_to_duplicate = match origin { hir::OpaqueTyOrigin::TyAlias { .. } => { - // in a TAIT like `type Foo<'a> = impl Foo<'a>`, we don't duplicate any - // lifetimes, since we don't have the issue that any are late-bound. - Vec::new() + // type alias impl trait and associated type position impl trait were + // decided to capture all in-scope lifetimes, which we collect for + // all opaques during resolution. + self.resolver + .take_extra_lifetime_params(opaque_ty_node_id) + .into_iter() + .map(|(ident, id, _)| Lifetime { id, ident }) + .collect() } hir::OpaqueTyOrigin::FnReturn(..) => { if matches!( @@ -1823,9 +1815,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { FnDeclKind::Fn | FnDeclKind::Inherent | FnDeclKind::Trait - | FnDeclKind::Impl => ImplTraitContext::ReturnPositionOpaqueTy { + | FnDeclKind::Impl => ImplTraitContext::OpaqueTy { origin: hir::OpaqueTyOrigin::FnReturn(self.local_def_id(fn_node_id)), - fn_kind: kind, + fn_kind: Some(kind), }, FnDeclKind::ExternFn => { ImplTraitContext::Disallowed(ImplTraitPosition::ExternFnReturn) @@ -1919,9 +1911,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { output, coro, opaque_ty_span, - ImplTraitContext::ReturnPositionOpaqueTy { + ImplTraitContext::OpaqueTy { origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id), - fn_kind, + fn_kind: Some(fn_kind), }, ); arena_vec![this; bound] diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs index 48941a232c2..aeeb4bf9e76 100644 --- a/compiler/rustc_ast_lowering/src/path.rs +++ b/compiler/rustc_ast_lowering/src/path.rs @@ -423,7 +423,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // fn f(_: impl Fn() -> impl Debug) -> impl Fn() -> impl Debug // // disallowed --^^^^^^^^^^ allowed --^^^^^^^^^^ // ``` - FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::ReturnPositionOpaqueTy { .. }) => { + FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::OpaqueTy { .. }) => { if self.tcx.features().impl_trait_in_fn_trait_return { self.lower_ty(ty, itctx) } else { diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 6119c6c84f8..e5d7b848903 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -1452,6 +1452,10 @@ impl<'a> State<'a> { s.print_path(&sym.path, true, 0); } } + InlineAsmOperand::Label { block } => { + s.head("label"); + s.print_block(block); + } } } AsmArg::ClobberAbi(abi) => { diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index ef582033c4e..5dc10308a34 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -723,7 +723,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro operands, options: _, line_spans: _, - destination: _, + targets: _, unwind: _, } => { for op in operands { @@ -749,7 +749,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro } InlineAsmOperand::Const { value: _ } | InlineAsmOperand::SymFn { value: _ } - | InlineAsmOperand::SymStatic { def_id: _ } => {} + | InlineAsmOperand::SymStatic { def_id: _ } + | InlineAsmOperand::Label { target_index: _ } => {} } } } diff --git a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs index 10941cadcbb..956de1dec9b 100644 --- a/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs +++ b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs @@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> { operands, options: _, line_spans: _, - destination: _, + targets: _, unwind: _, } => { for op in operands { @@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> { } InlineAsmOperand::Const { value: _ } | InlineAsmOperand::SymFn { value: _ } - | InlineAsmOperand::SymStatic { def_id: _ } => {} + | InlineAsmOperand::SymStatic { def_id: _ } + | InlineAsmOperand::Label { target_index: _ } => {} } } } diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs index 4b096a59234..12b02c7fcfa 100644 --- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs @@ -367,14 +367,17 @@ fn check_opaque_type_parameter_valid( span: Span, ) -> Result<(), ErrorGuaranteed> { let opaque_ty_hir = tcx.hir().expect_item(opaque_type_key.def_id); - let is_ty_alias = match opaque_ty_hir.expect_opaque_ty().origin { - OpaqueTyOrigin::TyAlias { .. } => true, - OpaqueTyOrigin::AsyncFn(..) | OpaqueTyOrigin::FnReturn(..) => false, + let (parent, is_ty_alias) = match opaque_ty_hir.expect_opaque_ty().origin { + OpaqueTyOrigin::TyAlias { parent, .. } => (parent, true), + OpaqueTyOrigin::AsyncFn(parent) | OpaqueTyOrigin::FnReturn(parent) => (parent, false), }; - let opaque_generics = tcx.generics_of(opaque_type_key.def_id); + let parent_generics = tcx.generics_of(parent); let mut seen_params: FxIndexMap<_, Vec<_>> = FxIndexMap::default(); - for (i, arg) in opaque_type_key.args.iter().enumerate() { + + // Only check the parent generics, which will ignore any of the + // duplicated lifetime args that come from reifying late-bounds. + for (i, arg) in opaque_type_key.args.iter().take(parent_generics.count()).enumerate() { if let Err(guar) = arg.error_reported() { return Err(guar); } @@ -395,7 +398,7 @@ fn check_opaque_type_parameter_valid( seen_params.entry(arg).or_default().push(i); } else { // Prevent `fn foo() -> Foo<u32>` from being defining. - let opaque_param = opaque_generics.param_at(i, tcx); + let opaque_param = parent_generics.param_at(i, tcx); let kind = opaque_param.kind.descr(); return Err(tcx.dcx().emit_err(NonGenericOpaqueTypeParam { @@ -409,10 +412,10 @@ fn check_opaque_type_parameter_valid( for (_, indices) in seen_params { if indices.len() > 1 { - let descr = opaque_generics.param_at(indices[0], tcx).kind.descr(); + let descr = parent_generics.param_at(indices[0], tcx).kind.descr(); let spans: Vec<_> = indices .into_iter() - .map(|i| tcx.def_span(opaque_generics.param_at(i, tcx).def_id)) + .map(|i| tcx.def_span(parent_generics.param_at(i, tcx).def_id)) .collect(); #[allow(rustc::diagnostic_outside_of_impl)] #[allow(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 0f3e995a331..8d38b86fa34 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1770,8 +1770,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { self.assert_iscleanup(body, block_data, real_target, is_cleanup); self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup); } - TerminatorKind::InlineAsm { destination, unwind, .. } => { - if let Some(target) = destination { + TerminatorKind::InlineAsm { ref targets, unwind, .. } => { + for &target in targets { self.assert_iscleanup(body, block_data, target, is_cleanup); } self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup); diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl index bc2a9d5ad1e..ba8401393d7 100644 --- a/compiler/rustc_builtin_macros/messages.ftl +++ b/compiler/rustc_builtin_macros/messages.ftl @@ -19,6 +19,8 @@ builtin_macros_asm_expected_other = expected operand, {$is_global_asm -> builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names +builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option + builtin_macros_asm_modifier_invalid = asm template modifier must be a single character builtin_macros_asm_mutually_exclusive = the `{$opt1}` and `{$opt2}` options are mutually exclusive diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index 38fa1ac5935..bc851fadc2e 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -164,6 +164,9 @@ pub fn parse_asm_args<'a>( path: path.clone(), }; ast::InlineAsmOperand::Sym { sym } + } else if !is_global_asm && p.eat_keyword(sym::label) { + let block = p.parse_block()?; + ast::InlineAsmOperand::Label { block } } else if allow_templates { let template = p.parse_expr()?; // If it can't possibly expand to a string, provide diagnostics here to include other @@ -240,6 +243,7 @@ pub fn parse_asm_args<'a>( let mut have_real_output = false; let mut outputs_sp = vec![]; let mut regclass_outputs = vec![]; + let mut labels_sp = vec![]; for (op, op_sp) in &args.operands { match op { ast::InlineAsmOperand::Out { reg, expr, .. } @@ -257,6 +261,9 @@ pub fn parse_asm_args<'a>( regclass_outputs.push(*op_sp); } } + ast::InlineAsmOperand::Label { .. } => { + labels_sp.push(*op_sp); + } _ => {} } } @@ -268,6 +275,9 @@ pub fn parse_asm_args<'a>( // Bail out now since this is likely to confuse MIR return Err(err); } + if args.options.contains(ast::InlineAsmOptions::MAY_UNWIND) && !labels_sp.is_empty() { + dcx.emit_err(errors::AsmMayUnwind { labels_sp }); + } if args.clobber_abis.len() > 0 { if is_global_asm { diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs index a2570e58713..06302ae577a 100644 --- a/compiler/rustc_builtin_macros/src/errors.rs +++ b/compiler/rustc_builtin_macros/src/errors.rs @@ -767,6 +767,13 @@ pub(crate) struct AsmNoReturn { } #[derive(Diagnostic)] +#[diag(builtin_macros_asm_mayunwind)] +pub(crate) struct AsmMayUnwind { + #[primary_span] + pub(crate) labels_sp: Vec<Span>, +} + +#[derive(Diagnostic)] #[diag(builtin_macros_global_asm_clobber_abi)] pub(crate) struct GlobalAsmClobberAbi { #[primary_span] diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index a7e76fbc128..1ce920f3bdb 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { template, operands, options, - destination, + targets, line_spans: _, unwind: _, } => { @@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { ); } + let have_labels = if options.contains(InlineAsmOptions::NORETURN) { + !targets.is_empty() + } else { + targets.len() > 1 + }; + if have_labels { + fx.tcx.dcx().span_fatal( + source_info.span, + "cranelift doesn't support labels in inline assembly.", + ); + } + crate::inline_asm::codegen_inline_asm_terminator( fx, source_info.span, template, operands, *options, - *destination, + targets.get(0).copied(), ); } TerminatorKind::UnwindTerminate(reason) => { diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index da07b66c762..44650898de8 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, InlineAsmOperand::In { .. } | InlineAsmOperand::Out { .. } | InlineAsmOperand::InOut { .. } - | InlineAsmOperand::SplitInOut { .. } => { + | InlineAsmOperand::SplitInOut { .. } + | InlineAsmOperand::Label { .. } => { span_bug!(op_sp, "invalid operand type for global_asm!") } } diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 7793b1b7092..171ee88a11c 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() } } + InlineAsmOperand::Label { .. } => { + span_bug!(span, "asm! label operands are not yet supported"); + } }) .collect::<Vec<_>>(); diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index 78e8e32b972..07edd26e27a 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -107,7 +107,7 @@ enum ConstraintOrRegister { impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { - fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) { + fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option<Self::BasicBlock>, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) { if options.contains(InlineAsmOptions::MAY_UNWIND) { self.sess().dcx() .create_err(UnwindingInlineAsm { span: span[0] }) @@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // added to `outputs.len()` let mut inputs = vec![]; + // GCC index of a label equals its position in the array added to + // `outputs.len() + inputs.len()`. + let mut labels = vec![]; + // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _` let mut clobbers = vec![]; @@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // some targets to add a leading underscore (Mach-O). constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); } + + InlineAsmOperandRef::Label { label } => { + labels.push(label); + } } } @@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { .. } => { // processed in the previous pass } + + InlineAsmOperandRef::Label { .. } => { + // processed in the previous pass + } } } @@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { ref string } => { template_str.push_str(string); } + + InlineAsmOperandRef::Label { label } => { + let label_gcc_index = labels.iter() + .position(|&l| l == label) + .expect("wrong rust index"); + let gcc_index = label_gcc_index + outputs.len() + inputs.len(); + push_to_template(Some('l'), gcc_index); + } } } } @@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // 4. Generate Extended Asm block let block = self.llbb(); - let extended_asm = block.add_extended_asm(None, &template_str); + let extended_asm = if let Some(dest) = dest { + assert!(!labels.is_empty()); + block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest)) + } else { + block.add_extended_asm(None, &template_str) + }; for op in &outputs { extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var); @@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { if !options.contains(InlineAsmOptions::NOSTACK) { // TODO(@Commeownist): figure out how to align stack } - if options.contains(InlineAsmOptions::NORETURN) { + if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) { let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable"); let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) }; self.call(self.type_void(), None, None, builtin_unreachable, &[], None); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a413466093b..74539d4d495 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option<Self::BasicBlock>, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Build the template string + let mut labels = vec![]; let mut template_str = String::new(); for piece in template { match *piece { @@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Only emit the raw symbol name template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } + InlineAsmOperandRef::Label { label } => { + template_str.push_str(&format!("${{{}:l}}", constraints.len())); + constraints.push("!i".to_owned()); + labels.push(label); + } } } } @@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &constraints.join(","), &inputs, output_type, + &labels, volatile, alignstack, dialect, line_spans, options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, + dest, + catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { + if let Some(dest) = dest { self.switch_to_block(dest); } @@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>( cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, + labels: &[&'ll llvm::BasicBlock], volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, + dest: Option<&'ll llvm::BasicBlock>, + catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; @@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>( can_throw, ); - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) + let call = if !labels.is_empty() { + assert!(catch_funclet.is_none()); + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) } else { bx.call(fty, None, None, v, inputs, None) }; diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index eaedaec635f..ca2e2b57580 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1538,6 +1538,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { } } + pub(crate) fn callbr( + &mut self, + llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + default_dest: &'ll BasicBlock, + indirect_dest: &[&'ll BasicBlock], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("callbr", llty, llfn, args); + let funclet_bundle = funclet.map(|funclet| funclet.bundle()); + let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } + + // Emit CFI pointer type membership test + self.cfi_type_test(fn_attrs, fn_abi, llfn); + + // Emit KCFI operand bundle + let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); + let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } + + let callbr = unsafe { + llvm::LLVMRustBuildCallBr( + self.llbuilder, + llty, + llfn, + default_dest, + indirect_dest.as_ptr(), + indirect_dest.len() as c_uint, + args.as_ptr(), + args.len() as c_uint, + bundles.as_ptr(), + bundles.len() as c_uint, + UNNAMED, + ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, callbr); + } + callbr + } + // Emits CFI pointer type membership tests. fn cfi_type_test( &mut self, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index f33a672aff0..467e02d55e3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -448,12 +448,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { constraint, inputs, self.type_void(), + &[], true, false, llvm::AsmDialect::Att, &[span], false, None, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 3a34ef0874e..58e98037067 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1616,6 +1616,20 @@ extern "C" { Name: *const c_char, ) -> &'a Value; + pub fn LLVMRustBuildCallBr<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + DefaultDest: &'a BasicBlock, + IndirectDests: *const &'a BasicBlock, + NumIndirectDests: c_uint, + Args: *const &'a Value, + NumArgs: c_uint, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); pub fn LLVMRustSetAlgebraicMath(Instr: &Value); pub fn LLVMRustSetAllowReassoc(Instr: &Value); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index b0efb646ef3..89a44d2897a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -264,7 +264,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { mir::UnwindAction::Unreachable => None, }; - if let Some(cleanup) = unwind_target { + if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) { + assert!(unwind_target.is_none()); let ret_llbb = if let Some(target) = destination { fx.llbb(target) } else { @@ -277,11 +278,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { options, line_spans, instance, - Some((ret_llbb, cleanup, self.funclet(fx))), + Some(ret_llbb), + None, + ); + MergingSucc::False + } else if let Some(cleanup) = unwind_target { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + operands, + options, + line_spans, + instance, + Some(ret_llbb), + Some((cleanup, self.funclet(fx))), ); MergingSucc::False } else { - bx.codegen_inline_asm(template, operands, options, line_spans, instance, None); + bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None); if let Some(target) = destination { self.funclet_br(fx, bx, target, mergeable_succ) @@ -1100,7 +1119,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands: &[mir::InlineAsmOperand<'tcx>], options: ast::InlineAsmOptions, line_spans: &[Span], - destination: Option<mir::BasicBlock>, + targets: &[mir::BasicBlock], unwind: mir::UnwindAction, instance: Instance<'_>, mergeable_succ: bool, @@ -1152,6 +1171,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::InlineAsmOperand::SymStatic { def_id } => { InlineAsmOperandRef::SymStatic { def_id } } + mir::InlineAsmOperand::Label { target_index } => { + InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) } + } }) .collect(); @@ -1162,7 +1184,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &operands, options, line_spans, - destination, + if options.contains(InlineAsmOptions::NORETURN) { + None + } else { + targets.get(0).copied() + }, unwind, instance, mergeable_succ, @@ -1318,7 +1344,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ref operands, options, line_spans, - destination, + ref targets, unwind, } => self.codegen_asm_terminator( helper, @@ -1328,7 +1354,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands, options, line_spans, - destination, + targets, unwind, self.instance, mergeable_succ(), diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index 295e2769109..1a4795c0213 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -76,7 +76,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> { hir::InlineAsmOperand::In { .. } | hir::InlineAsmOperand::Out { .. } | hir::InlineAsmOperand::InOut { .. } - | hir::InlineAsmOperand::SplitInOut { .. } => { + | hir::InlineAsmOperand::SplitInOut { .. } + | hir::InlineAsmOperand::Label { .. } => { span_bug!(*op_sp, "invalid operand type for global_asm!") } }) diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index c2ae74b18d8..8d67b626bbd 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { SymStatic { def_id: DefId, }, + Label { + label: B::BasicBlock, + }, } #[derive(Debug)] @@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option<Self::BasicBlock>, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ); } diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl index 2805ca360ad..81c9e02e2ee 100644 --- a/compiler/rustc_const_eval/messages.ftl +++ b/compiler/rustc_const_eval/messages.ftl @@ -232,9 +232,6 @@ const_eval_non_const_fn_call = const_eval_non_const_impl = impl defined here, but it is not `const` -const_eval_noreturn_asm_returned = - returned from noreturn inline assembly - const_eval_not_enough_caller_args = calling a function with fewer arguments than it requires diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index ee5de44a651..c798f1ce018 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -374,11 +374,17 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized { kind: Option<MemoryKind<Self::MemoryKind>>, ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>; + /// Evaluate the inline assembly. + /// + /// This should take care of jumping to the next block (one of `targets`) when asm goto + /// is triggered, `targets[0]` when the assembly falls through, or diverge in case of + /// `InlineAsmOptions::NORETURN` being set. fn eval_inline_asm( _ecx: &mut InterpCx<'mir, 'tcx, Self>, _template: &'tcx [InlineAsmTemplatePiece], _operands: &[mir::InlineAsmOperand<'tcx>], _options: InlineAsmOptions, + _targets: &[mir::BasicBlock], ) -> InterpResult<'tcx> { throw_unsup_format!("inline assembly is not supported") } diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index 55f85573f38..f2b1ec42567 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -1,6 +1,5 @@ use std::borrow::Cow; -use rustc_ast::ast::InlineAsmOptions; use rustc_middle::{ mir, ty::{ @@ -224,15 +223,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { terminator.kind ), - InlineAsm { template, ref operands, options, destination, .. } => { - M::eval_inline_asm(self, template, operands, options)?; - if options.contains(InlineAsmOptions::NORETURN) { - throw_ub_custom!(fluent::const_eval_noreturn_asm_returned); - } - self.go_to_block( - destination - .expect("InlineAsm terminators without noreturn must have a destination"), - ) + InlineAsm { template, ref operands, options, ref targets, .. } => { + M::eval_inline_asm(self, template, operands, options, targets)?; } } diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index cc49e8ea247..74ba2f6039e 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -471,9 +471,9 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *real_target, EdgeKind::Normal); self.check_unwind_edge(location, *unwind); } - TerminatorKind::InlineAsm { destination, unwind, .. } => { - if let Some(destination) = destination { - self.check_edge(location, *destination, EdgeKind::Normal); + TerminatorKind::InlineAsm { targets, unwind, .. } => { + for &target in targets { + self.check_edge(location, target, EdgeKind::Normal); } self.check_unwind_edge(location, *unwind); } diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index 9f4bead21dc..a10f4b934ea 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -345,6 +345,8 @@ declare_features! ( (unstable, asm_const, "1.58.0", Some(93332)), /// Enables experimental inline assembly support for additional architectures. (unstable, asm_experimental_arch, "1.58.0", Some(93335)), + /// Allows using `label` operands in inline assembly. + (unstable, asm_goto, "CURRENT_RUSTC_VERSION", Some(119364)), /// Allows the `may_unwind` option in inline assembly. (unstable, asm_unwind, "1.58.0", Some(93334)), /// Allows users to enforce equality of associated constants `TraitImpl<AssocConst=3>`. diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 3dac6880f17..7d5b4c0f06d 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -2562,6 +2562,8 @@ pub enum OpaqueTyOrigin { AsyncFn(LocalDefId), /// type aliases: `type Foo = impl Trait;` TyAlias { + /// The type alias or associated type parent of the TAIT/ATPIT + parent: LocalDefId, /// associated types in impl blocks for traits. in_assoc_ty: bool, }, @@ -2650,6 +2652,9 @@ pub enum InlineAsmOperand<'hir> { path: QPath<'hir>, def_id: DefId, }, + Label { + block: &'hir Block<'hir>, + }, } impl<'hir> InlineAsmOperand<'hir> { @@ -2659,7 +2664,10 @@ impl<'hir> InlineAsmOperand<'hir> { | Self::Out { reg, .. } | Self::InOut { reg, .. } | Self::SplitInOut { reg, .. } => Some(reg), - Self::Const { .. } | Self::SymFn { .. } | Self::SymStatic { .. } => None, + Self::Const { .. } + | Self::SymFn { .. } + | Self::SymStatic { .. } + | Self::Label { .. } => None, } } @@ -2680,6 +2688,12 @@ pub struct InlineAsm<'hir> { pub line_spans: &'hir [Span], } +impl InlineAsm<'_> { + pub fn contains_label(&self) -> bool { + self.operands.iter().any(|x| matches!(x.0, InlineAsmOperand::Label { .. })) + } +} + /// Represents a parameter in a function header. #[derive(Debug, Clone, Copy, HashStable_Generic)] pub struct Param<'hir> { diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs index 88e0238b523..32b4bf38fa9 100644 --- a/compiler/rustc_hir/src/intravisit.rs +++ b/compiler/rustc_hir/src/intravisit.rs @@ -1289,6 +1289,7 @@ pub fn walk_inline_asm<'v, V: Visitor<'v>>( InlineAsmOperand::SymStatic { path, .. } => { try_visit!(visitor.visit_qpath(path, id, *op_sp)); } + InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)), } } V::Result::output() diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index 748571c12b3..1b24c2b61fd 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -339,8 +339,9 @@ fn check_opaque_meets_bounds<'tcx>( origin: &hir::OpaqueTyOrigin, ) -> Result<(), ErrorGuaranteed> { let defining_use_anchor = match *origin { - hir::OpaqueTyOrigin::FnReturn(did) | hir::OpaqueTyOrigin::AsyncFn(did) => did, - hir::OpaqueTyOrigin::TyAlias { .. } => tcx.impl_trait_parent(def_id), + hir::OpaqueTyOrigin::FnReturn(did) + | hir::OpaqueTyOrigin::AsyncFn(did) + | hir::OpaqueTyOrigin::TyAlias { parent: did, .. } => did, }; let param_env = tcx.param_env(defining_use_anchor); @@ -351,14 +352,14 @@ fn check_opaque_meets_bounds<'tcx>( let ocx = ObligationCtxt::new(&infcx); let args = match *origin { - hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent) => { - GenericArgs::identity_for_item(tcx, parent).extend_to( - tcx, - def_id.to_def_id(), - |param, _| tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local()).into(), - ) - } - hir::OpaqueTyOrigin::TyAlias { .. } => GenericArgs::identity_for_item(tcx, def_id), + hir::OpaqueTyOrigin::FnReturn(parent) + | hir::OpaqueTyOrigin::AsyncFn(parent) + | hir::OpaqueTyOrigin::TyAlias { parent, .. } => GenericArgs::identity_for_item( + tcx, parent, + ) + .extend_to(tcx, def_id.to_def_id(), |param, _| { + tcx.map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local()).into() + }), }; let opaque_ty = Ty::new_opaque(tcx, def_id.to_def_id(), args); diff --git a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs index d03b02f028d..9de660407d7 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs @@ -470,6 +470,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> { } }; } + // No special checking is needed for labels. + hir::InlineAsmOperand::Label { .. } => {} } } } diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs index f70bb8c4289..2675eacc06e 100644 --- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs @@ -339,7 +339,7 @@ fn compute_bidirectional_outlives_predicates<'tcx>( predicates: &mut Vec<(ty::Clause<'tcx>, Span)>, ) { for param in opaque_own_params { - let orig_lifetime = tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local()); + let orig_lifetime = tcx.map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local()); if let ty::ReEarlyParam(..) = *orig_lifetime { let dup_lifetime = ty::Region::new_early_param( tcx, diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs index ad8ec1036ef..607ada8b5ed 100644 --- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs +++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs @@ -514,38 +514,11 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> { // These sorts of items have no lifetime parameters at all. intravisit::walk_item(self, item); } - hir::ItemKind::OpaqueTy(hir::OpaqueTy { - origin: hir::OpaqueTyOrigin::TyAlias { .. }, - .. - }) => { - // Opaque types are visited when we visit the - // `TyKind::OpaqueDef`, so that they have the lifetimes from - // their parent opaque_ty in scope. - // - // The core idea here is that since OpaqueTys are generated with the impl Trait as - // their owner, we can keep going until we find the Item that owns that. We then - // conservatively add all resolved lifetimes. Otherwise we run into problems in - // cases like `type Foo<'a> = impl Bar<As = impl Baz + 'a>`. - let parent_item = self.tcx.hir().get_parent_item(item.hir_id()); - let resolved_lifetimes: &ResolveBoundVars = - self.tcx.resolve_bound_vars(parent_item); - // We need to add *all* deps, since opaque tys may want them from *us* - for (&owner, defs) in resolved_lifetimes.defs.iter() { - defs.iter().for_each(|(&local_id, region)| { - self.map.defs.insert(hir::HirId { owner, local_id }, *region); - }); - } - for (&owner, late_bound_vars) in resolved_lifetimes.late_bound_vars.iter() { - late_bound_vars.iter().for_each(|(&local_id, late_bound_vars)| { - self.record_late_bound_vars( - hir::HirId { owner, local_id }, - late_bound_vars.clone(), - ); - }); - } - } hir::ItemKind::OpaqueTy(&hir::OpaqueTy { - origin: hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent), + origin: + hir::OpaqueTyOrigin::FnReturn(parent) + | hir::OpaqueTyOrigin::AsyncFn(parent) + | hir::OpaqueTyOrigin::TyAlias { parent, .. }, generics, .. }) => { @@ -683,26 +656,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> { // the opaque_ty generics let opaque_ty = self.tcx.hir().item(item_id); match &opaque_ty.kind { - hir::ItemKind::OpaqueTy(hir::OpaqueTy { - origin: hir::OpaqueTyOrigin::TyAlias { .. }, - .. - }) => { - intravisit::walk_ty(self, ty); - - // Elided lifetimes and late-bound lifetimes (from the parent) - // are not allowed in non-return position impl Trait - let scope = Scope::LateBoundary { - s: &Scope::TraitRefBoundary { s: self.scope }, - what: "type alias impl trait", - }; - self.with(scope, |this| intravisit::walk_item(this, opaque_ty)); - - return; - } - hir::ItemKind::OpaqueTy(hir::OpaqueTy { - origin: hir::OpaqueTyOrigin::FnReturn(..) | hir::OpaqueTyOrigin::AsyncFn(..), - .. - }) => {} + hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin: _, .. }) => {} i => bug!("`impl Trait` pointed to non-opaque type?? {:#?}", i), }; diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs index 2217e5280a7..3d54dd6f9f6 100644 --- a/compiler/rustc_hir_analysis/src/collect/type_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs @@ -553,11 +553,11 @@ pub(super) fn type_of_opaque( Ok(ty::EarlyBinder::bind(match tcx.hir_node_by_def_id(def_id) { Node::Item(item) => match item.kind { ItemKind::OpaqueTy(OpaqueTy { - origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: false }, + origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: false, .. }, .. }) => opaque::find_opaque_ty_constraints_for_tait(tcx, def_id), ItemKind::OpaqueTy(OpaqueTy { - origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: true }, + origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: true, .. }, .. }) => opaque::find_opaque_ty_constraints_for_impl_trait_in_assoc_type(tcx, def_id), // Opaque types desugared from `impl Trait`. diff --git a/compiler/rustc_hir_analysis/src/variance/mod.rs b/compiler/rustc_hir_analysis/src/variance/mod.rs index 9618c6d8011..42885b0c832 100644 --- a/compiler/rustc_hir_analysis/src/variance/mod.rs +++ b/compiler/rustc_hir_analysis/src/variance/mod.rs @@ -125,15 +125,7 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc // By default, RPIT are invariant wrt type and const generics, but they are bivariant wrt // lifetime generics. - let variances = std::iter::repeat(ty::Invariant).take(generics.count()); - - let mut variances: Vec<_> = match tcx.opaque_type_origin(item_def_id) { - rustc_hir::OpaqueTyOrigin::FnReturn(_) | rustc_hir::OpaqueTyOrigin::AsyncFn(_) => { - variances.collect() - } - // But TAIT are invariant for all generics - rustc_hir::OpaqueTyOrigin::TyAlias { .. } => return tcx.arena.alloc_from_iter(variances), - }; + let mut variances = vec![ty::Invariant; generics.count()]; // Mark all lifetimes from parent generics as unused (Bivariant). // This will be overridden later if required. diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 8f8f747339b..b5bb063c5ed 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -1265,6 +1265,10 @@ impl<'a> State<'a> { s.space(); s.print_qpath(path, true); } + hir::InlineAsmOperand::Label { block } => { + s.head("label"); + s.print_block(block); + } }, AsmArg::Options(opts) => { s.word("options"); diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index fcb490bcfec..ceaae9cf49f 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -3232,6 +3232,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>) -> Ty<'tcx> { + let mut diverge = asm.options.contains(ast::InlineAsmOptions::NORETURN); + for (op, _op_sp) in asm.operands { match op { hir::InlineAsmOperand::In { expr, .. } => { @@ -3253,13 +3255,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // be well-formed. hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } => {} hir::InlineAsmOperand::SymStatic { .. } => {} + hir::InlineAsmOperand::Label { block } => { + let previous_diverges = self.diverges.get(); + + // The label blocks should have unit return value or diverge. + let ty = + self.check_block_with_expected(block, ExpectHasType(self.tcx.types.unit)); + if !ty.is_never() { + self.demand_suptype(block.span, self.tcx.types.unit, ty); + diverge = false; + } + + // We need this to avoid false unreachable warning when a label diverges. + self.diverges.set(previous_diverges); + } } } - if asm.options.contains(ast::InlineAsmOptions::NORETURN) { - self.tcx.types.never - } else { - Ty::new_unit(self.tcx) - } + + if diverge { self.tcx.types.never } else { self.tcx.types.unit } } fn check_offset_of( diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs index 04fb7bcf4f3..ba0383d19b9 100644 --- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs +++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs @@ -293,6 +293,9 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> { | hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } | hir::InlineAsmOperand::SymStatic { .. } => {} + hir::InlineAsmOperand::Label { block } => { + self.walk_block(block); + } } } } diff --git a/compiler/rustc_infer/src/infer/opaque_types/mod.rs b/compiler/rustc_infer/src/infer/opaque_types/mod.rs index b6f3c38cb3f..3d6829ba657 100644 --- a/compiler/rustc_infer/src/infer/opaque_types/mod.rs +++ b/compiler/rustc_infer/src/infer/opaque_types/mod.rs @@ -391,7 +391,7 @@ impl<'tcx> InferCtxt<'tcx> { // Anonymous `impl Trait` hir::OpaqueTyOrigin::FnReturn(parent) => parent == parent_def_id, // Named `type Foo = impl Bar;` - hir::OpaqueTyOrigin::TyAlias { in_assoc_ty } => { + hir::OpaqueTyOrigin::TyAlias { in_assoc_ty, .. } => { if in_assoc_ty { self.tcx.opaque_types_defined_by(parent_def_id).contains(&def_id) } else { diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 1cdfc22431e..f6253068eaa 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -451,14 +451,30 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.ObjectFilenameForDebug = OutputObjFile; } if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib; +#else Options.CompressDebugSections = DebugCompressionType::Zlib; +#endif } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd; +#else Options.CompressDebugSections = DebugCompressionType::Zstd; +#endif } else if (!strcmp("none", DebugInfoCompression)) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::None; +#else Options.CompressDebugSections = DebugCompressionType::None; +#endif } +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.X86RelaxRelocations = RelaxELFRelocations; +#else Options.RelaxELFRelocations = RelaxELFRelocations; +#endif Options.UseInitArray = UseInitArray; #if LLVM_VERSION_LT(17, 0) diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index d9262a92782..3e998d428ee 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1549,6 +1549,31 @@ LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, Name)); } +extern "C" LLVMValueRef +LLVMRustBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMBasicBlockRef DefaultDest, + LLVMBasicBlockRef *IndirectDests, unsigned NumIndirectDests, + LLVMValueRef *Args, unsigned NumArgs, + OperandBundleDef **OpBundles, unsigned NumOpBundles, + const char *Name) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap<FunctionType>(Ty); + + // FIXME: Is there a way around this? + std::vector<BasicBlock*> IndirectDestsUnwrapped; + IndirectDestsUnwrapped.reserve(NumIndirectDests); + for (unsigned i = 0; i < NumIndirectDests; ++i) { + IndirectDestsUnwrapped.push_back(unwrap(IndirectDests[i])); + } + + return wrap(unwrap(B)->CreateCallBr( + FTy, Callee, unwrap(DefaultDest), + ArrayRef<BasicBlock*>(IndirectDestsUnwrapped), + ArrayRef<Value*>(unwrap(Args), NumArgs), + ArrayRef<OperandBundleDef>(*OpBundles, NumOpBundles), + Name)); +} + extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, LLVMBasicBlockRef BB) { auto Point = unwrap(BB)->getFirstInsertionPt(); diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 4b97c25f610..0221bc6c363 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1716,13 +1716,13 @@ mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; // tidy-alphabetical-start - static_assert_size!(BasicBlockData<'_>, 136); + static_assert_size!(BasicBlockData<'_>, 144); static_assert_size!(LocalDecl<'_>, 40); static_assert_size!(SourceScopeData<'_>, 72); static_assert_size!(Statement<'_>, 32); static_assert_size!(StatementKind<'_>, 16); - static_assert_size!(Terminator<'_>, 104); - static_assert_size!(TerminatorKind<'_>, 88); + static_assert_size!(Terminator<'_>, 112); + static_assert_size!(TerminatorKind<'_>, 96); static_assert_size!(VarDebugInfo<'_>, 88); // tidy-alphabetical-end } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 5638b575b31..e058302af31 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use crate::mir::interpret::ConstAllocation; use super::graphviz::write_mir_fn_graphviz; -use rustc_ast::InlineAsmTemplatePiece; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_middle::mir::interpret::{ alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, GlobalAlloc, Pointer, Provenance, @@ -830,6 +830,9 @@ impl<'tcx> TerminatorKind<'tcx> { InlineAsmOperand::SymStatic { def_id } => { write!(fmt, "sym_static {def_id:?}")?; } + InlineAsmOperand::Label { target_index } => { + write!(fmt, "label {target_index}")?; + } } } write!(fmt, ", options({options:?}))") @@ -868,16 +871,19 @@ impl<'tcx> TerminatorKind<'tcx> { vec!["real".into(), "unwind".into()] } FalseUnwind { unwind: _, .. } => vec!["real".into()], - InlineAsm { destination: Some(_), unwind: UnwindAction::Cleanup(_), .. } => { - vec!["return".into(), "unwind".into()] - } - InlineAsm { destination: Some(_), unwind: _, .. } => { - vec!["return".into()] - } - InlineAsm { destination: None, unwind: UnwindAction::Cleanup(_), .. } => { - vec!["unwind".into()] + InlineAsm { options, ref targets, unwind, .. } => { + let mut vec = Vec::with_capacity(targets.len() + 1); + if !options.contains(InlineAsmOptions::NORETURN) { + vec.push("return".into()); + } + vec.resize(targets.len(), "label".into()); + + if let UnwindAction::Cleanup(_) = unwind { + vec.push("unwind".into()); + } + + vec } - InlineAsm { destination: None, unwind: _, .. } => vec![], } } } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 5c9857b9c53..f188923f876 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -793,9 +793,10 @@ pub enum TerminatorKind<'tcx> { /// used to map assembler errors back to the line in the source code. line_spans: &'tcx [Span], - /// Destination block after the inline assembly returns, unless it is - /// diverging (InlineAsmOptions::NORETURN). - destination: Option<BasicBlock>, + /// Valid targets for the inline assembly. + /// The first element is the fallthrough destination, unless + /// InlineAsmOptions::NORETURN is set. + targets: Vec<BasicBlock>, /// Action to be taken if the inline assembly unwinds. This is present /// if and only if InlineAsmOptions::MAY_UNWIND is set. @@ -918,6 +919,10 @@ pub enum InlineAsmOperand<'tcx> { SymStatic { def_id: DefId, }, + Label { + /// This represents the index into the `targets` array in `TerminatorKind::InlineAsm`. + target_index: usize, + }, } /// Type for MIR `Assert` terminator error messages. diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index ae3daa77b1c..a4a4df55c48 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -347,8 +347,7 @@ pub struct Terminator<'tcx> { } pub type Successors<'a> = impl DoubleEndedIterator<Item = BasicBlock> + 'a; -pub type SuccessorsMut<'a> = - iter::Chain<std::option::IntoIter<&'a mut BasicBlock>, slice::IterMut<'a, BasicBlock>>; +pub type SuccessorsMut<'a> = impl DoubleEndedIterator<Item = &'a mut BasicBlock> + 'a; impl<'tcx> Terminator<'tcx> { #[inline] @@ -382,40 +381,36 @@ impl<'tcx> TerminatorKind<'tcx> { pub fn successors(&self) -> Successors<'_> { use self::TerminatorKind::*; match *self { - Call { target: Some(t), unwind: UnwindAction::Cleanup(ref u), .. } - | Yield { resume: t, drop: Some(ref u), .. } - | Drop { target: t, unwind: UnwindAction::Cleanup(ref u), .. } - | Assert { target: t, unwind: UnwindAction::Cleanup(ref u), .. } - | FalseUnwind { real_target: t, unwind: UnwindAction::Cleanup(ref u) } - | InlineAsm { destination: Some(t), unwind: UnwindAction::Cleanup(ref u), .. } => { - Some(t).into_iter().chain(slice::from_ref(u).into_iter().copied()) - } - Goto { target: t } - | Call { target: None, unwind: UnwindAction::Cleanup(t), .. } - | Call { target: Some(t), unwind: _, .. } - | Yield { resume: t, drop: None, .. } - | Drop { target: t, unwind: _, .. } - | Assert { target: t, unwind: _, .. } - | FalseUnwind { real_target: t, unwind: _ } - | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. } - | InlineAsm { destination: Some(t), unwind: _, .. } => { - Some(t).into_iter().chain((&[]).into_iter().copied()) + Call { target: Some(ref t), unwind: UnwindAction::Cleanup(u), .. } + | Yield { resume: ref t, drop: Some(u), .. } + | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), .. } + | Assert { target: ref t, unwind: UnwindAction::Cleanup(u), .. } + | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) } => { + slice::from_ref(t).into_iter().copied().chain(Some(u)) + } + Goto { target: ref t } + | Call { target: None, unwind: UnwindAction::Cleanup(ref t), .. } + | Call { target: Some(ref t), unwind: _, .. } + | Yield { resume: ref t, drop: None, .. } + | Drop { target: ref t, unwind: _, .. } + | Assert { target: ref t, unwind: _, .. } + | FalseUnwind { real_target: ref t, unwind: _ } => { + slice::from_ref(t).into_iter().copied().chain(None) } UnwindResume | UnwindTerminate(_) | CoroutineDrop | Return | Unreachable - | Call { target: None, unwind: _, .. } - | InlineAsm { destination: None, unwind: _, .. } => { - None.into_iter().chain((&[]).into_iter().copied()) + | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None), + InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => { + targets.iter().copied().chain(Some(u)) } - SwitchInt { ref targets, .. } => { - None.into_iter().chain(targets.targets.iter().copied()) + InlineAsm { ref targets, unwind: _, .. } => targets.iter().copied().chain(None), + SwitchInt { ref targets, .. } => targets.targets.iter().copied().chain(None), + FalseEdge { ref real_target, imaginary_target } => { + slice::from_ref(real_target).into_iter().copied().chain(Some(imaginary_target)) } - FalseEdge { real_target, ref imaginary_target } => Some(real_target) - .into_iter() - .chain(slice::from_ref(imaginary_target).into_iter().copied()), } } @@ -427,33 +422,31 @@ impl<'tcx> TerminatorKind<'tcx> { | Yield { resume: ref mut t, drop: Some(ref mut u), .. } | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. } | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. } - | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) } - | InlineAsm { - destination: Some(ref mut t), - unwind: UnwindAction::Cleanup(ref mut u), - .. - } => Some(t).into_iter().chain(slice::from_mut(u)), + | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) } => { + slice::from_mut(t).into_iter().chain(Some(u)) + } Goto { target: ref mut t } | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. } | Call { target: Some(ref mut t), unwind: _, .. } | Yield { resume: ref mut t, drop: None, .. } | Drop { target: ref mut t, unwind: _, .. } | Assert { target: ref mut t, unwind: _, .. } - | FalseUnwind { real_target: ref mut t, unwind: _ } - | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. } - | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => { - Some(t).into_iter().chain(&mut []) + | FalseUnwind { real_target: ref mut t, unwind: _ } => { + slice::from_mut(t).into_iter().chain(None) } UnwindResume | UnwindTerminate(_) | CoroutineDrop | Return | Unreachable - | Call { target: None, unwind: _, .. } - | InlineAsm { destination: None, unwind: _, .. } => None.into_iter().chain(&mut []), - SwitchInt { ref mut targets, .. } => None.into_iter().chain(&mut targets.targets), + | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None), + InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => { + targets.iter_mut().chain(Some(u)) + } + InlineAsm { ref mut targets, unwind: _, .. } => targets.iter_mut().chain(None), + SwitchInt { ref mut targets, .. } => targets.targets.iter_mut().chain(None), FalseEdge { ref mut real_target, ref mut imaginary_target } => { - Some(real_target).into_iter().chain(slice::from_mut(imaginary_target)) + slice::from_mut(real_target).into_iter().chain(Some(imaginary_target)) } } } @@ -525,7 +518,7 @@ pub enum TerminatorEdges<'mir, 'tcx> { Double(BasicBlock, BasicBlock), /// Special action for `Yield`, `Call` and `InlineAsm` terminators. AssignOnReturn { - return_: Option<BasicBlock>, + return_: &'mir [BasicBlock], /// The cleanup block, if it exists. cleanup: Option<BasicBlock>, place: CallReturnPlaces<'mir, 'tcx>, @@ -589,31 +582,37 @@ impl<'tcx> TerminatorKind<'tcx> { TerminatorEdges::Double(real_target, imaginary_target) } - Yield { resume: target, drop, resume_arg, value: _ } => { + Yield { resume: ref target, drop, resume_arg, value: _ } => { TerminatorEdges::AssignOnReturn { - return_: Some(target), + return_: slice::from_ref(target), cleanup: drop, place: CallReturnPlaces::Yield(resume_arg), } } - Call { unwind, destination, target, func: _, args: _, fn_span: _, call_source: _ } => { - TerminatorEdges::AssignOnReturn { - return_: target, - cleanup: unwind.cleanup_block(), - place: CallReturnPlaces::Call(destination), - } - } + Call { + unwind, + destination, + ref target, + func: _, + args: _, + fn_span: _, + call_source: _, + } => TerminatorEdges::AssignOnReturn { + return_: target.as_ref().map(slice::from_ref).unwrap_or_default(), + cleanup: unwind.cleanup_block(), + place: CallReturnPlaces::Call(destination), + }, InlineAsm { template: _, ref operands, options: _, line_spans: _, - destination, + ref targets, unwind, } => TerminatorEdges::AssignOnReturn { - return_: destination, + return_: targets, cleanup: unwind.cleanup_block(), place: CallReturnPlaces::InlineAsm(operands), }, diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 2c5ca82a4cd..845b1717550 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -565,7 +565,7 @@ macro_rules! make_mir_visitor { operands, options: _, line_spans: _, - destination: _, + targets: _, unwind: _, } => { for op in operands { @@ -595,7 +595,8 @@ macro_rules! make_mir_visitor { self.visit_constant(value, location); } InlineAsmOperand::Out { place: None, .. } - | InlineAsmOperand::SymStatic { def_id: _ } => {} + | InlineAsmOperand::SymStatic { def_id: _ } + | InlineAsmOperand::Label { target_index: _ } => {} } } } diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs index 65875ff3f9a..913d0072c29 100644 --- a/compiler/rustc_middle/src/thir.rs +++ b/compiler/rustc_middle/src/thir.rs @@ -565,6 +565,9 @@ pub enum InlineAsmOperand<'tcx> { SymStatic { def_id: DefId, }, + Label { + block: BlockId, + }, } #[derive(Copy, Clone, Debug, PartialEq, HashStable)] diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs index 4847a7bea91..5952c296fb6 100644 --- a/compiler/rustc_middle/src/thir/visit.rs +++ b/compiler/rustc_middle/src/thir/visit.rs @@ -162,6 +162,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>( | Const { value: _, span: _ } | SymFn { value: _, span: _ } | SymStatic { def_id: _ } => {} + Label { block } => visitor.visit_block(&visitor.thir()[*block]), } } } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 4f8e4aa90f4..4a04e677180 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1254,7 +1254,7 @@ impl<'tcx> TyCtxt<'tcx> { if self.def_kind(scope) == DefKind::OpaqueTy { // Lifetime params of opaque types are synthetic and thus irrelevant to // diagnostics. Map them back to their origin! - region = self.map_rpit_lifetime_to_fn_lifetime(def_id); + region = self.map_opaque_lifetime_to_parent_lifetime(def_id); continue; } break (scope, ty::BrNamed(def_id.into(), self.item_name(def_id.into()))); @@ -2219,31 +2219,31 @@ impl<'tcx> TyCtxt<'tcx> { ) } - /// Given the def-id of an early-bound lifetime on an RPIT corresponding to + /// Given the def-id of an early-bound lifetime on an opaque corresponding to /// a duplicated captured lifetime, map it back to the early- or late-bound /// lifetime of the function from which it originally as captured. If it is /// a late-bound lifetime, this will represent the liberated (`ReLateParam`) lifetime /// of the signature. // FIXME(RPITIT): if we ever synthesize new lifetimes for RPITITs and not just // re-use the generics of the opaque, this function will need to be tweaked slightly. - pub fn map_rpit_lifetime_to_fn_lifetime( + pub fn map_opaque_lifetime_to_parent_lifetime( self, - mut rpit_lifetime_param_def_id: LocalDefId, + mut opaque_lifetime_param_def_id: LocalDefId, ) -> ty::Region<'tcx> { debug_assert!( - matches!(self.def_kind(rpit_lifetime_param_def_id), DefKind::LifetimeParam), - "{rpit_lifetime_param_def_id:?} is a {}", - self.def_descr(rpit_lifetime_param_def_id.to_def_id()) + matches!(self.def_kind(opaque_lifetime_param_def_id), DefKind::LifetimeParam), + "{opaque_lifetime_param_def_id:?} is a {}", + self.def_descr(opaque_lifetime_param_def_id.to_def_id()) ); loop { - let parent = self.local_parent(rpit_lifetime_param_def_id); + let parent = self.local_parent(opaque_lifetime_param_def_id); let hir::OpaqueTy { lifetime_mapping, .. } = self.hir_node_by_def_id(parent).expect_item().expect_opaque_ty(); let Some((lifetime, _)) = lifetime_mapping .iter() - .find(|(_, duplicated_param)| *duplicated_param == rpit_lifetime_param_def_id) + .find(|(_, duplicated_param)| *duplicated_param == opaque_lifetime_param_def_id) else { bug!("duplicated lifetime param should be present"); }; @@ -2256,7 +2256,7 @@ impl<'tcx> TyCtxt<'tcx> { // of the opaque we mapped from. Continue mapping. if matches!(self.def_kind(new_parent), DefKind::OpaqueTy) { debug_assert_eq!(self.parent(parent.to_def_id()), new_parent); - rpit_lifetime_param_def_id = ebv.expect_local(); + opaque_lifetime_param_def_id = ebv.expect_local(); continue; } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 4edec0e6951..5c2d3973d61 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1940,18 +1940,6 @@ impl<'tcx> TyCtxt<'tcx> { matches!(self.trait_of_item(def_id), Some(trait_id) if self.has_attr(trait_id, sym::const_trait)) } - /// Returns the `DefId` of the item within which the `impl Trait` is declared. - /// For type-alias-impl-trait this is the `type` alias. - /// For impl-trait-in-assoc-type this is the assoc type. - /// For return-position-impl-trait this is the function. - pub fn impl_trait_parent(self, mut def_id: LocalDefId) -> LocalDefId { - // Find the surrounding item (type alias or assoc type) - while let DefKind::OpaqueTy = self.def_kind(def_id) { - def_id = self.local_parent(def_id); - } - def_id - } - pub fn impl_method_has_trait_impl_trait_tys(self, def_id: DefId) -> bool { if self.def_kind(def_id) != DefKind::AssocFn { return false; diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs index 8a9dc34bc7b..b4eeeccc127 100644 --- a/compiler/rustc_mir_build/src/build/expr/into.rs +++ b/compiler/rustc_mir_build/src/build/expr/into.rs @@ -383,6 +383,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { line_spans, }) => { use rustc_middle::{mir, thir}; + + let destination_block = this.cfg.start_new_block(); + let mut targets = if options.contains(InlineAsmOptions::NORETURN) { + vec![] + } else { + vec![destination_block] + }; + let operands = operands .into_iter() .map(|op| match *op { @@ -438,14 +446,28 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { thir::InlineAsmOperand::SymStatic { def_id } => { mir::InlineAsmOperand::SymStatic { def_id } } + thir::InlineAsmOperand::Label { block } => { + let target = this.cfg.start_new_block(); + let target_index = targets.len(); + targets.push(target); + + let tmp = this.get_unit_temp(); + let target = unpack!(this.ast_block(tmp, target, block, source_info)); + this.cfg.terminate( + target, + source_info, + TerminatorKind::Goto { target: destination_block }, + ); + + mir::InlineAsmOperand::Label { target_index } + } }) .collect(); - if !options.contains(InlineAsmOptions::NORETURN) { + if !expr.ty.is_never() { this.cfg.push_assign_unit(block, source_info, destination, this.tcx); } - let destination_block = this.cfg.start_new_block(); this.cfg.terminate( block, source_info, @@ -454,11 +476,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { operands, options, line_spans, - destination: if options.contains(InlineAsmOptions::NORETURN) { - None - } else { - Some(destination_block) - }, + targets, unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) { UnwindAction::Continue } else { diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs index 3f2b7c482a6..2c817d605af 100644 --- a/compiler/rustc_mir_build/src/lints.rs +++ b/compiler/rustc_mir_build/src/lints.rs @@ -199,9 +199,10 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx | TerminatorKind::Unreachable | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive), - // A diverging InlineAsm is treated as non-recursing - TerminatorKind::InlineAsm { destination, .. } => { - if destination.is_some() { + // A InlineAsm without targets (diverging and contains no labels) + // is treated as non-recursing. + TerminatorKind::InlineAsm { ref targets, .. } => { + if !targets.is_empty() { ControlFlow::Continue(()) } else { ControlFlow::Break(NonRecursive) diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs index 62762168cf4..2318e84292b 100644 --- a/compiler/rustc_mir_build/src/thir/cx/expr.rs +++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs @@ -656,6 +656,9 @@ impl<'tcx> Cx<'tcx> { hir::InlineAsmOperand::SymStatic { path: _, def_id } => { InlineAsmOperand::SymStatic { def_id } } + hir::InlineAsmOperand::Label { block } => { + InlineAsmOperand::Label { block: self.mirror_block(block) } + } }) .collect(), options: asm.options, diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs index 267ea3aa3e1..d53704f89e7 100644 --- a/compiler/rustc_mir_build/src/thir/print.rs +++ b/compiler/rustc_mir_build/src/thir/print.rs @@ -889,6 +889,12 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> { print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1); print_indented!(self, "}", depth_lvl + 1); } + InlineAsmOperand::Label { block } => { + print_indented!(self, "InlineAsmOperand::Block {", depth_lvl); + print_indented!(self, "block:", depth_lvl + 1); + self.print_block(*block, depth_lvl + 2); + print_indented!(self, "}", depth_lvl + 1); + } } } } diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index 4c3fadf487b..f57e8b8bd6f 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -242,9 +242,9 @@ impl Direction for Backward { propagate(pred, &tmp); } - mir::TerminatorKind::InlineAsm { - destination: Some(dest), ref operands, .. - } if dest == bb => { + mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. } + if targets.contains(&bb) => + { let mut tmp = exit_state.clone(); analysis.apply_call_return_effect( &mut tmp, @@ -491,9 +491,12 @@ impl Direction for Forward { if let Some(cleanup) = cleanup { propagate(cleanup, exit_state); } - if let Some(return_) = return_ { + + if !return_.is_empty() { analysis.apply_call_return_effect(exit_state, bb, place); - propagate(return_, exit_state); + for &target in return_ { + propagate(target, exit_state); + } } } TerminatorEdges::SwitchInt { targets, discr } => { diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs index 0270e059a58..a827f6a8dbd 100644 --- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs +++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs @@ -299,7 +299,9 @@ where })?; } - mir::TerminatorKind::InlineAsm { destination: Some(_), ref operands, .. } => { + mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. } + if !targets.is_empty() => + { self.write_row(w, "", "(on successful return)", |this, w, fmt| { let state_on_unwind = this.results.get().clone(); this.results.apply_custom_effect(|analysis, state| { diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs index 595c2ff5bf7..29169c31263 100644 --- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs @@ -271,7 +271,8 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> { InlineAsmOperand::In { .. } | InlineAsmOperand::Const { .. } | InlineAsmOperand::SymFn { .. } - | InlineAsmOperand::SymStatic { .. } => {} + | InlineAsmOperand::SymStatic { .. } + | InlineAsmOperand::Label { .. } => {} } } } diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index c6ec1b5aee4..db48ecd702b 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -491,7 +491,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> { ref operands, options: _, line_spans: _, - destination: _, + targets: _, unwind: _, } => { for op in operands { @@ -515,7 +515,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> { } InlineAsmOperand::Const { value: _ } | InlineAsmOperand::SymFn { value: _ } - | InlineAsmOperand::SymStatic { def_id: _ } => {} + | InlineAsmOperand::SymStatic { def_id: _ } + | InlineAsmOperand::Label { target_index: _ } => {} } } } diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index c97192435ce..ed8c4d8283d 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -349,12 +349,20 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera | FalseUnwind { real_target: target, .. } | Goto { target } => CoverageSuccessors::Chainable(target), - // These terminators can normally be chained, except when they have no + // A call terminator can normally be chained, except when they have no // successor because they are known to diverge. - Call { target: maybe_target, .. } | InlineAsm { destination: maybe_target, .. } => { - match maybe_target { - Some(target) => CoverageSuccessors::Chainable(target), - None => CoverageSuccessors::NotChainable(&[]), + Call { target: maybe_target, .. } => match maybe_target { + Some(target) => CoverageSuccessors::Chainable(target), + None => CoverageSuccessors::NotChainable(&[]), + }, + + // An inline asm terminator can normally be chained, except when it diverges or uses asm + // goto. + InlineAsm { ref targets, .. } => { + if targets.len() == 1 { + CoverageSuccessors::Chainable(targets[0]) + } else { + CoverageSuccessors::NotChainable(targets) } } diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index d9a3c0cb162..569998de35e 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -88,7 +88,6 @@ impl<'tcx> MockBlocks<'tcx> { | TerminatorKind::FalseEdge { real_target: ref mut target, .. } | TerminatorKind::FalseUnwind { real_target: ref mut target, .. } | TerminatorKind::Goto { ref mut target } - | TerminatorKind::InlineAsm { destination: Some(ref mut target), .. } | TerminatorKind::Yield { resume: ref mut target, .. } => *target = to_block, ref invalid => bug!("Invalid from_block: {:?}", invalid), } @@ -185,10 +184,12 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String { | TerminatorKind::FalseEdge { real_target: target, .. } | TerminatorKind::FalseUnwind { real_target: target, .. } | TerminatorKind::Goto { target } - | TerminatorKind::InlineAsm { destination: Some(target), .. } | TerminatorKind::Yield { resume: target, .. } => { format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), target) } + TerminatorKind::InlineAsm { targets, .. } => { + format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets) + } TerminatorKind::SwitchInt { targets, .. } => { format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets) } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 2c8201b1903..10fea09531a 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -648,7 +648,8 @@ impl WriteInfo { } InlineAsmOperand::Const { .. } | InlineAsmOperand::SymFn { .. } - | InlineAsmOperand::SymStatic { .. } => (), + | InlineAsmOperand::SymStatic { .. } + | InlineAsmOperand::Label { .. } => {} } } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 36546a03cdf..f6a0945c222 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -1036,8 +1036,8 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { { bug!("False unwinds should have been removed before inlining") } - TerminatorKind::InlineAsm { ref mut destination, ref mut unwind, .. } => { - if let Some(ref mut tgt) = *destination { + TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => { + for tgt in targets.iter_mut() { *tgt = self.map_block(*tgt); } *unwind = self.map_unwind(*unwind); diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 37cce625c8e..32f823a5ac6 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -446,7 +446,8 @@ fn collect_items_rec<'tcx>( hir::InlineAsmOperand::In { .. } | hir::InlineAsmOperand::Out { .. } | hir::InlineAsmOperand::InOut { .. } - | hir::InlineAsmOperand::SplitInOut { .. } => { + | hir::InlineAsmOperand::SplitInOut { .. } + | hir::InlineAsmOperand::Label { .. } => { span_bug!(*op_sp, "invalid operand type for global_asm!") } } diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index ea9c78ca34c..73f5829adec 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -448,7 +448,7 @@ impl<'a> Parser<'a> { } /// Parses a block. No inner attributes are allowed. - pub(super) fn parse_block(&mut self) -> PResult<'a, P<Block>> { + pub fn parse_block(&mut self) -> PResult<'a, P<Block>> { let (attrs, block) = self.parse_inner_attrs_and_block()?; if let [.., last] = &*attrs { self.error_on_forbidden_inner_attr( diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs index 487407014d1..e5033e1f51f 100644 --- a/compiler/rustc_passes/src/liveness.rs +++ b/compiler/rustc_passes/src/liveness.rs @@ -86,7 +86,6 @@ use crate::errors; use self::LiveNodeKind::*; use self::VarKind::*; -use rustc_ast::InlineAsmOptions; use rustc_data_structures::fx::FxIndexMap; use rustc_hir as hir; use rustc_hir::def::*; @@ -416,6 +415,12 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> { self.add_live_node_for_node(expr.hir_id, ExprNode(expr.span, expr.hir_id)); } + // Inline assembly may contain labels. + hir::ExprKind::InlineAsm(asm) if asm.contains_label() => { + self.add_live_node_for_node(expr.hir_id, ExprNode(expr.span, expr.hir_id)); + intravisit::walk_expr(self, expr); + } + // otherwise, live nodes are not required: hir::ExprKind::Index(..) | hir::ExprKind::Field(..) @@ -1045,20 +1050,53 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> { | hir::ExprKind::Repeat(ref e, _) => self.propagate_through_expr(e, succ), hir::ExprKind::InlineAsm(asm) => { + // + // (inputs) + // | + // v + // (outputs) + // / \ + // | | + // v v + // (labels)(fallthrough) + // | | + // v v + // ( succ / exit_ln ) + // Handle non-returning asm - let mut succ = if asm.options.contains(InlineAsmOptions::NORETURN) { - self.exit_ln - } else { - succ - }; + let mut succ = + if self.typeck_results.expr_ty(expr).is_never() { self.exit_ln } else { succ }; + + // Do a first pass for labels only + if asm.contains_label() { + let ln = self.live_node(expr.hir_id, expr.span); + self.init_from_succ(ln, succ); + for (op, _op_sp) in asm.operands.iter().rev() { + match op { + hir::InlineAsmOperand::Label { block } => { + let label_ln = self.propagate_through_block(block, succ); + self.merge_from_succ(ln, label_ln); + } + hir::InlineAsmOperand::In { .. } + | hir::InlineAsmOperand::Out { .. } + | hir::InlineAsmOperand::InOut { .. } + | hir::InlineAsmOperand::SplitInOut { .. } + | hir::InlineAsmOperand::Const { .. } + | hir::InlineAsmOperand::SymFn { .. } + | hir::InlineAsmOperand::SymStatic { .. } => {} + } + } + succ = ln; + } - // Do a first pass for writing outputs only + // Do a second pass for writing outputs only for (op, _op_sp) in asm.operands.iter().rev() { match op { hir::InlineAsmOperand::In { .. } | hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } - | hir::InlineAsmOperand::SymStatic { .. } => {} + | hir::InlineAsmOperand::SymStatic { .. } + | hir::InlineAsmOperand::Label { .. } => {} hir::InlineAsmOperand::Out { expr, .. } => { if let Some(expr) = expr { succ = self.write_place(expr, succ, ACC_WRITE); @@ -1075,7 +1113,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> { } } - // Then do a second pass for inputs + // Then do a third pass for inputs for (op, _op_sp) in asm.operands.iter().rev() { match op { hir::InlineAsmOperand::In { expr, .. } => { @@ -1097,7 +1135,8 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> { } hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } - | hir::InlineAsmOperand::SymStatic { .. } => {} + | hir::InlineAsmOperand::SymStatic { .. } + | hir::InlineAsmOperand::Label { .. } => {} } } succ diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_passes/src/naked_functions.rs index 0455d6d4acb..27c9c1306e6 100644 --- a/compiler/rustc_passes/src/naked_functions.rs +++ b/compiler/rustc_passes/src/naked_functions.rs @@ -237,7 +237,8 @@ impl<'tcx> CheckInlineAssembly<'tcx> { InlineAsmOperand::In { .. } | InlineAsmOperand::Out { .. } | InlineAsmOperand::InOut { .. } - | InlineAsmOperand::SplitInOut { .. } => Some(op_sp), + | InlineAsmOperand::SplitInOut { .. } + | InlineAsmOperand::Label { .. } => Some(op_sp), }) .collect(); if !unsupported_operands.is_empty() { diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 455afb8de8e..d2aea0056b3 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -1242,6 +1242,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast, self.resolve_anon_const(anon_const, AnonConstKind::InlineConst); } InlineAsmOperand::Sym { sym } => self.visit_inline_asm_sym(sym), + InlineAsmOperand::Label { block } => self.visit_block(block), } } } diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs index 501d6f7d304..003a9a59200 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs @@ -559,7 +559,8 @@ impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> { } InlineAsmOperand::Const { .. } | InlineAsmOperand::SymFn { .. } - | InlineAsmOperand::SymStatic { .. } => (None, None), + | InlineAsmOperand::SymStatic { .. } + | InlineAsmOperand::Label { .. } => (None, None), }; stable_mir::mir::InlineAsmOperand { in_value, out_place, raw_rpr: format!("{self:?}") } @@ -632,14 +633,15 @@ impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> { operands, options, line_spans, - destination, + targets, unwind, } => TerminatorKind::InlineAsm { template: format!("{template:?}"), operands: operands.iter().map(|operand| operand.stable(tables)).collect(), options: format!("{options:?}"), line_spans: format!("{line_spans:?}"), - destination: destination.map(|d| d.as_usize()), + // FIXME: Figure out how to do labels in SMIR + destination: targets.first().map(|d| d.as_usize()), unwind: unwind.stable(tables), }, mir::TerminatorKind::Yield { .. } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 9e628e4ef91..9dadd471247 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -399,6 +399,7 @@ symbols! { asm, asm_const, asm_experimental_arch, + asm_goto, asm_sym, asm_unwind, assert, diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index a68dae0ade3..2e66d4d0ab0 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -1404,6 +1404,7 @@ supported_targets! { ("i686-unknown-linux-gnu", i686_unknown_linux_gnu), ("i586-unknown-linux-gnu", i586_unknown_linux_gnu), ("loongarch64-unknown-linux-gnu", loongarch64_unknown_linux_gnu), + ("loongarch64-unknown-linux-musl", loongarch64_unknown_linux_musl), ("m68k-unknown-linux-gnu", m68k_unknown_linux_gnu), ("csky-unknown-linux-gnuabiv2", csky_unknown_linux_gnuabiv2), ("csky-unknown-linux-gnuabiv2hf", csky_unknown_linux_gnuabiv2hf), diff --git a/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs new file mode 100644 index 00000000000..9f653174cf0 --- /dev/null +++ b/compiler/rustc_target/src/spec/targets/loongarch64_unknown_linux_musl.rs @@ -0,0 +1,19 @@ +use crate::spec::{base, Target, TargetOptions}; + +pub fn target() -> Target { + Target { + llvm_target: "loongarch64-unknown-linux-musl".into(), + description: None, + pointer_width: 64, + data_layout: "e-m:e-p:64:64-i64:64-i128:128-n64-S128".into(), + arch: "loongarch64".into(), + options: TargetOptions { + cpu: "generic".into(), + features: "+f,+d".into(), + llvm_abiname: "lp64d".into(), + max_atomic_width: Some(64), + crt_static_default: false, + ..base::linux_musl::opts() + }, + } +} diff --git a/compiler/rustc_ty_utils/src/implied_bounds.rs b/compiler/rustc_ty_utils/src/implied_bounds.rs index 191671bcc1e..87462963c27 100644 --- a/compiler/rustc_ty_utils/src/implied_bounds.rs +++ b/compiler/rustc_ty_utils/src/implied_bounds.rs @@ -71,7 +71,7 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<' // the end of the list corresponding to the opaque's generics. for param in &generics.params[tcx.generics_of(fn_def_id).params.len()..] { let orig_lt = - tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local()); + tcx.map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local()); if matches!(*orig_lt, ty::ReLateParam(..)) { mapping.insert( orig_lt, diff --git a/compiler/rustc_ty_utils/src/opaque_types.rs b/compiler/rustc_ty_utils/src/opaque_types.rs index c6448b4f661..3dcc8382289 100644 --- a/compiler/rustc_ty_utils/src/opaque_types.rs +++ b/compiler/rustc_ty_utils/src/opaque_types.rs @@ -141,7 +141,7 @@ impl<'tcx> OpaqueTypeCollector<'tcx> { trace!(?origin); match origin { rustc_hir::OpaqueTyOrigin::FnReturn(_) | rustc_hir::OpaqueTyOrigin::AsyncFn(_) => {} - rustc_hir::OpaqueTyOrigin::TyAlias { in_assoc_ty } => { + rustc_hir::OpaqueTyOrigin::TyAlias { in_assoc_ty, .. } => { if !in_assoc_ty { if !self.check_tait_defining_scope(alias_ty.def_id.expect_local()) { return; |