diff options
Diffstat (limited to 'compiler')
| -rw-r--r-- | compiler/rustc_builtin_macros/messages.ftl | 1 | ||||
| -rw-r--r-- | compiler/rustc_builtin_macros/src/autodiff.rs | 67 | ||||
| -rw-r--r-- | compiler/rustc_builtin_macros/src/errors.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_builtin_macros/src/lib.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 53 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 80 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/block.rs | 329 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/intrinsic.rs | 34 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/traits/intrinsic.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_expand/src/expand.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_mir_build/src/builder/scope.rs | 112 | ||||
| -rw-r--r-- | compiler/rustc_parse/src/parser/stmt.rs | 13 | ||||
| -rw-r--r-- | compiler/rustc_passes/src/check_attr.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_resolve/src/diagnostics.rs | 50 | ||||
| -rw-r--r-- | compiler/rustc_span/src/symbol.rs | 5 |
15 files changed, 408 insertions, 364 deletions
diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl index 628bdee1129..d32e6f1558e 100644 --- a/compiler/rustc_builtin_macros/messages.ftl +++ b/compiler/rustc_builtin_macros/messages.ftl @@ -56,7 +56,6 @@ builtin_macros_assert_requires_expression = macro requires an expression as an a builtin_macros_autodiff = autodiff must be applied to function builtin_macros_autodiff_missing_config = autodiff requires at least a name and mode -builtin_macros_autodiff_mode = unknown Mode: `{$mode}`. Use `Forward` or `Reverse` builtin_macros_autodiff_mode_activity = {$act} can not be used in {$mode} Mode builtin_macros_autodiff_not_build = this rustc version does not support autodiff builtin_macros_autodiff_number_activities = expected {$expected} activities, but found {$found} diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index 1ff4fc6aaab..dc3bb8ab52a 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -86,27 +86,23 @@ mod llvm_enzyme { ecx: &mut ExtCtxt<'_>, meta_item: &ThinVec<MetaItemInner>, has_ret: bool, + mode: DiffMode, ) -> AutoDiffAttrs { let dcx = ecx.sess.dcx(); - let mode = name(&meta_item[1]); - let Ok(mode) = DiffMode::from_str(&mode) else { - dcx.emit_err(errors::AutoDiffInvalidMode { span: meta_item[1].span(), mode }); - return AutoDiffAttrs::error(); - }; // Now we check, whether the user wants autodiff in batch/vector mode, or scalar mode. // If he doesn't specify an integer (=width), we default to scalar mode, thus width=1. - let mut first_activity = 2; + let mut first_activity = 1; - let width = if let [_, _, x, ..] = &meta_item[..] + let width = if let [_, x, ..] = &meta_item[..] && let Some(x) = width(x) { - first_activity = 3; + first_activity = 2; match x.try_into() { Ok(x) => x, Err(_) => { dcx.emit_err(errors::AutoDiffInvalidWidth { - span: meta_item[2].span(), + span: meta_item[1].span(), width: x, }); return AutoDiffAttrs::error(); @@ -165,6 +161,24 @@ mod llvm_enzyme { ts.push(TokenTree::Token(comma.clone(), Spacing::Alone)); } + pub(crate) fn expand_forward( + ecx: &mut ExtCtxt<'_>, + expand_span: Span, + meta_item: &ast::MetaItem, + item: Annotatable, + ) -> Vec<Annotatable> { + expand_with_mode(ecx, expand_span, meta_item, item, DiffMode::Forward) + } + + pub(crate) fn expand_reverse( + ecx: &mut ExtCtxt<'_>, + expand_span: Span, + meta_item: &ast::MetaItem, + item: Annotatable, + ) -> Vec<Annotatable> { + expand_with_mode(ecx, expand_span, meta_item, item, DiffMode::Reverse) + } + /// We expand the autodiff macro to generate a new placeholder function which passes /// type-checking and can be called by users. The function body of the placeholder function will /// later be replaced on LLVM-IR level, so the design of the body is less important and for now @@ -198,11 +212,12 @@ mod llvm_enzyme { /// ``` /// FIXME(ZuseZ4): Once autodiff is enabled by default, make this a doc comment which is checked /// in CI. - pub(crate) fn expand( + pub(crate) fn expand_with_mode( ecx: &mut ExtCtxt<'_>, expand_span: Span, meta_item: &ast::MetaItem, mut item: Annotatable, + mode: DiffMode, ) -> Vec<Annotatable> { if cfg!(not(llvm_enzyme)) { ecx.sess.dcx().emit_err(errors::AutoDiffSupportNotBuild { span: meta_item.span }); @@ -245,29 +260,41 @@ mod llvm_enzyme { // create TokenStream from vec elemtents: // meta_item doesn't have a .tokens field let mut ts: Vec<TokenTree> = vec![]; - if meta_item_vec.len() < 2 { - // At the bare minimum, we need a fnc name and a mode, even for a dummy function with no - // input and output args. + if meta_item_vec.len() < 1 { + // At the bare minimum, we need a fnc name. dcx.emit_err(errors::AutoDiffMissingConfig { span: item.span() }); return vec![item]; } - meta_item_inner_to_ts(&meta_item_vec[1], &mut ts); + let mode_symbol = match mode { + DiffMode::Forward => sym::Forward, + DiffMode::Reverse => sym::Reverse, + _ => unreachable!("Unsupported mode: {:?}", mode), + }; + + // Insert mode token + let mode_token = Token::new(TokenKind::Ident(mode_symbol, false.into()), Span::default()); + ts.insert(0, TokenTree::Token(mode_token, Spacing::Joint)); + ts.insert( + 1, + TokenTree::Token(Token::new(TokenKind::Comma, Span::default()), Spacing::Alone), + ); // Now, if the user gave a width (vector aka batch-mode ad), then we copy it. // If it is not given, we default to 1 (scalar mode). let start_position; let kind: LitKind = LitKind::Integer; let symbol; - if meta_item_vec.len() >= 3 - && let Some(width) = width(&meta_item_vec[2]) + if meta_item_vec.len() >= 2 + && let Some(width) = width(&meta_item_vec[1]) { - start_position = 3; + start_position = 2; symbol = Symbol::intern(&width.to_string()); } else { - start_position = 2; + start_position = 1; symbol = sym::integer(1); } + let l: Lit = Lit { kind, symbol, suffix: None }; let t = Token::new(TokenKind::Literal(l), Span::default()); let comma = Token::new(TokenKind::Comma, Span::default()); @@ -289,7 +316,7 @@ mod llvm_enzyme { ts.pop(); let ts: TokenStream = TokenStream::from_iter(ts); - let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret); + let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret, mode); if !x.is_active() { // We encountered an error, so we return the original item. // This allows us to potentially parse other attributes. @@ -1017,4 +1044,4 @@ mod llvm_enzyme { } } -pub(crate) use llvm_enzyme::expand; +pub(crate) use llvm_enzyme::{expand_forward, expand_reverse}; diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs index 75d06a8df14..3a2e96a5e5a 100644 --- a/compiler/rustc_builtin_macros/src/errors.rs +++ b/compiler/rustc_builtin_macros/src/errors.rs @@ -181,14 +181,6 @@ mod autodiff { } #[derive(Diagnostic)] - #[diag(builtin_macros_autodiff_mode)] - pub(crate) struct AutoDiffInvalidMode { - #[primary_span] - pub(crate) span: Span, - pub(crate) mode: String, - } - - #[derive(Diagnostic)] #[diag(builtin_macros_autodiff_width)] pub(crate) struct AutoDiffInvalidWidth { #[primary_span] diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs index 9cd4d17059a..b16f3cff5cf 100644 --- a/compiler/rustc_builtin_macros/src/lib.rs +++ b/compiler/rustc_builtin_macros/src/lib.rs @@ -5,10 +5,10 @@ #![allow(internal_features)] #![allow(rustc::diagnostic_outside_of_impl)] #![allow(rustc::untranslatable_diagnostic)] +#![cfg_attr(not(bootstrap), feature(autodiff))] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] #![feature(assert_matches)] -#![feature(autodiff)] #![feature(box_patterns)] #![feature(decl_macro)] #![feature(if_let_guard)] @@ -112,7 +112,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { register_attr! { alloc_error_handler: alloc_error_handler::expand, - autodiff: autodiff::expand, + autodiff_forward: autodiff::expand_forward, + autodiff_reverse: autodiff::expand_reverse, bench: test::expand_bench, cfg_accessible: cfg_accessible::Expander, cfg_eval: cfg_eval::expand, diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index ba65c8205a5..1bcb891a250 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -22,11 +22,11 @@ use rustc_codegen_ssa::traits::{ }; use rustc_middle::bug; #[cfg(feature = "master")] -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, Ty}; use rustc_span::{Span, Symbol, sym}; -use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, PassMode}; use rustc_target::spec::PanicStrategy; #[cfg(feature = "master")] @@ -200,9 +200,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc fn codegen_intrinsic_call( &mut self, instance: Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], - llresult: RValue<'gcc>, + result: PlaceRef<'tcx, RValue<'gcc>>, span: Span, ) -> Result<(), Instance<'tcx>> { let tcx = self.tcx; @@ -221,7 +220,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc let name_str = name.as_str(); let llret_ty = self.layout_of(ret_ty).gcc_type(self); - let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); let simple = get_simple_intrinsic(self, name); let simple_func = get_simple_function(self, name); @@ -271,7 +269,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc args[0].immediate(), args[1].immediate(), args[2].immediate(), - llresult, + result, ); return Ok(()); } @@ -286,17 +284,10 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } sym::volatile_load | sym::unaligned_volatile_load => { - let tp_ty = fn_args.type_at(0); let ptr = args[0].immediate(); - let layout = self.layout_of(tp_ty); - let load = if let PassMode::Cast { cast: ref ty, pad_i32: _ } = fn_abi.ret.mode { - let gcc_ty = ty.gcc_type(self); - self.volatile_load(gcc_ty, ptr) - } else { - self.volatile_load(layout.gcc_type(self), ptr) - }; + let load = self.volatile_load(result.layout.gcc_type(self), ptr); // TODO(antoyo): set alignment. - if let BackendRepr::Scalar(scalar) = layout.backend_repr { + if let BackendRepr::Scalar(scalar) = result.layout.backend_repr { self.to_immediate_scalar(load, scalar) } else { load @@ -511,16 +502,14 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc _ => return Err(Instance::new_raw(instance.def_id(), instance.args)), }; - if !fn_abi.ret.is_ignore() { - if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode { - let ptr_llty = self.type_ptr_to(ty.gcc_type(self)); - let ptr = self.pointercast(result.val.llval, ptr_llty); - self.store(value, ptr, result.val.align); - } else { - OperandRef::from_immediate_or_packed_pair(self, value, result.layout) - .val - .store(self, result); - } + if result.layout.ty.is_bool() { + OperandRef::from_immediate_or_packed_pair(self, value, result.layout) + .val + .store(self, result); + } else if !result.layout.ty.is_unit() { + let ptr_llty = self.type_ptr_to(result.layout.gcc_type(self)); + let ptr = self.pointercast(result.val.llval, ptr_llty); + self.store(value, ptr, result.val.align); } Ok(()) } @@ -1230,14 +1219,13 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( try_func: RValue<'gcc>, data: RValue<'gcc>, _catch_func: RValue<'gcc>, - dest: RValue<'gcc>, + dest: PlaceRef<'tcx, RValue<'gcc>>, ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { bx.call(bx.type_void(), None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. - let ret_align = bx.tcx.data_layout.i32_align.abi; - bx.store(bx.const_i32(0), dest, ret_align); + OperandValue::Immediate(bx.const_i32(0)).store(bx, dest); } else { if wants_msvc_seh(bx.sess()) { unimplemented!(); @@ -1261,12 +1249,12 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( // functions in play. By calling a shim we're guaranteed that our shim will have // the right personality function. #[cfg(feature = "master")] -fn codegen_gnu_try<'gcc>( - bx: &mut Builder<'_, 'gcc, '_>, +fn codegen_gnu_try<'gcc, 'tcx>( + bx: &mut Builder<'_, 'gcc, 'tcx>, try_func: RValue<'gcc>, data: RValue<'gcc>, catch_func: RValue<'gcc>, - dest: RValue<'gcc>, + dest: PlaceRef<'tcx, RValue<'gcc>>, ) { let cx: &CodegenCx<'gcc, '_> = bx.cx; let (llty, func) = get_rust_try_fn(cx, &mut |mut bx| { @@ -1322,8 +1310,7 @@ fn codegen_gnu_try<'gcc>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); + OperandValue::Immediate(ret).store(bx, dest); } // Helper function used to get a handle to the `__rust_try` function used to diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 5ca57375292..e8629aeebb9 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -15,11 +15,10 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; use rustc_symbol_mangling::mangle_internal_symbol; -use rustc_target::callconv::{FnAbi, PassMode}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use tracing::debug; -use crate::abi::{FnAbiLlvmExt, LlvmType}; +use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, Metadata}; @@ -165,9 +164,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, &'ll Value>], - llresult: &'ll Value, + result: PlaceRef<'tcx, &'ll Value>, span: Span, ) -> Result<(), ty::Instance<'tcx>> { let tcx = self.tcx; @@ -184,7 +182,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let name = tcx.item_name(def_id); let llret_ty = self.layout_of(ret_ty).llvm_type(self); - let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); let simple = get_simple_intrinsic(self, name); let llval = match name { @@ -255,7 +252,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { args[0].immediate(), args[1].immediate(), args[2].immediate(), - llresult, + result, ); return Ok(()); } @@ -264,7 +261,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) } sym::va_arg => { - match fn_abi.ret.layout.backend_repr { + match result.layout.backend_repr { BackendRepr::Scalar(scalar) => { match scalar.primitive() { Primitive::Int(..) => { @@ -299,18 +296,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::volatile_load | sym::unaligned_volatile_load => { - let tp_ty = fn_args.type_at(0); let ptr = args[0].immediate(); - let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode { - let llty = ty.llvm_type(self); - self.volatile_load(llty, ptr) - } else { - self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) - }; + let load = self.volatile_load(result.layout.llvm_type(self), ptr); let align = if name == sym::unaligned_volatile_load { 1 } else { - self.align_of(tp_ty).bytes() as u32 + result.layout.align.abi.bytes() as u32 }; unsafe { llvm::LLVMSetAlignment(load, align); @@ -629,14 +620,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } }; - if !fn_abi.ret.is_ignore() { - if let PassMode::Cast { .. } = &fn_abi.ret.mode { - self.store(llval, result.val.llval, result.val.align); - } else { - OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) - .val - .store(self, result); - } + if result.layout.ty.is_bool() { + OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) + .val + .store(self, result); + } else if !result.layout.ty.is_unit() { + self.store_to_place(llval, result.val); } Ok(()) } @@ -688,20 +677,19 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } -fn catch_unwind_intrinsic<'ll>( - bx: &mut Builder<'_, 'll, '_>, +fn catch_unwind_intrinsic<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, - dest: &'ll Value, + dest: PlaceRef<'tcx, &'ll Value>, ) { if bx.sess().panic_strategy() == PanicStrategy::Abort { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.call(try_func_ty, None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. - let ret_align = bx.tcx().data_layout.i32_align.abi; - bx.store(bx.const_i32(0), dest, ret_align); + OperandValue::Immediate(bx.const_i32(0)).store(bx, dest); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); } else if wants_wasm_eh(bx.sess()) { @@ -720,12 +708,12 @@ fn catch_unwind_intrinsic<'ll>( // instructions are meant to work for all targets, as of the time of this // writing, however, LLVM does not recommend the usage of these new instructions // as the old ones are still more optimized. -fn codegen_msvc_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, +fn codegen_msvc_try<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, - dest: &'ll Value, + dest: PlaceRef<'tcx, &'ll Value>, ) { let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { bx.set_personality_fn(bx.eh_personality()); @@ -865,17 +853,16 @@ fn codegen_msvc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); + OperandValue::Immediate(ret).store(bx, dest); } // WASM's definition of the `rust_try` function. -fn codegen_wasm_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, +fn codegen_wasm_try<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, - dest: &'ll Value, + dest: PlaceRef<'tcx, &'ll Value>, ) { let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { bx.set_personality_fn(bx.eh_personality()); @@ -939,8 +926,7 @@ fn codegen_wasm_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); + OperandValue::Immediate(ret).store(bx, dest); } // Definition of the standard `try` function for Rust using the GNU-like model @@ -954,12 +940,12 @@ fn codegen_wasm_try<'ll>( // function calling it, and that function may already have other personality // functions in play. By calling a shim we're guaranteed that our shim will have // the right personality function. -fn codegen_gnu_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, +fn codegen_gnu_try<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, - dest: &'ll Value, + dest: PlaceRef<'tcx, &'ll Value>, ) { let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { // Codegens the shims described above: @@ -1006,19 +992,18 @@ fn codegen_gnu_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); + OperandValue::Immediate(ret).store(bx, dest); } // Variant of codegen_gnu_try used for emscripten where Rust panics are // implemented using C++ exceptions. Here we use exceptions of a specific type // (`struct rust_panic`) to represent Rust panics. -fn codegen_emcc_try<'ll>( - bx: &mut Builder<'_, 'll, '_>, +fn codegen_emcc_try<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, - dest: &'ll Value, + dest: PlaceRef<'tcx, &'ll Value>, ) { let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { // Codegens the shims described above: @@ -1089,8 +1074,7 @@ fn codegen_emcc_try<'ll>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); - let i32_align = bx.tcx().data_layout.i32_align.abi; - bx.store(ret, dest, i32_align); + OperandValue::Immediate(ret).store(bx, dest); } // Helper function to give a Block to a closure to codegen a shim function. diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 922b8a5824b..1baab62ae43 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -11,8 +11,8 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; +use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_span::{Span, sym}; use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; use tracing::{debug, info}; @@ -827,7 +827,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { helper: &TerminatorCodegenHelper<'tcx>, bx: &mut Bx, intrinsic: ty::IntrinsicDef, - instance: Option<Instance<'tcx>>, + instance: Instance<'tcx>, source_info: mir::SourceInfo, target: Option<mir::BasicBlock>, unwind: mir::UnwindAction, @@ -836,58 +836,56 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Emit a panic or a no-op for `assert_*` intrinsics. // These are intrinsics that compile to panics so that we can get a message // which mentions the offending type, even from a const context. - if let Some(requirement) = ValidityRequirement::from_intrinsic(intrinsic.name) { - let ty = instance.unwrap().args.type_at(0); - - let do_panic = !bx - .tcx() - .check_validity_requirement((requirement, bx.typing_env().as_query_input(ty))) - .expect("expect to have layout during codegen"); - - let layout = bx.layout_of(ty); - - Some(if do_panic { - let msg_str = with_no_visible_paths!({ - with_no_trimmed_paths!({ - if layout.is_uninhabited() { - // Use this error even for the other intrinsics as it is more precise. - format!("attempted to instantiate uninhabited type `{ty}`") - } else if requirement == ValidityRequirement::Zero { - format!("attempted to zero-initialize type `{ty}`, which is invalid") - } else { - format!( - "attempted to leave type `{ty}` uninitialized, which is invalid" - ) - } - }) - }); - let msg = bx.const_str(&msg_str); + let Some(requirement) = ValidityRequirement::from_intrinsic(intrinsic.name) else { + return None; + }; - // Obtain the panic entry point. - let (fn_abi, llfn, instance) = - common::build_langcall(bx, Some(source_info.span), LangItem::PanicNounwind); + let ty = instance.args.type_at(0); - // Codegen the actual panic invoke/call. - helper.do_call( - self, - bx, - fn_abi, - llfn, - &[msg.0, msg.1], - target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), - unwind, - &[], - Some(instance), - mergeable_succ, - ) - } else { - // a NOP - let target = target.unwrap(); - helper.funclet_br(self, bx, target, mergeable_succ) - }) - } else { - None + let is_valid = bx + .tcx() + .check_validity_requirement((requirement, bx.typing_env().as_query_input(ty))) + .expect("expect to have layout during codegen"); + + if is_valid { + // a NOP + let target = target.unwrap(); + return Some(helper.funclet_br(self, bx, target, mergeable_succ)); } + + let layout = bx.layout_of(ty); + + let msg_str = with_no_visible_paths!({ + with_no_trimmed_paths!({ + if layout.is_uninhabited() { + // Use this error even for the other intrinsics as it is more precise. + format!("attempted to instantiate uninhabited type `{ty}`") + } else if requirement == ValidityRequirement::Zero { + format!("attempted to zero-initialize type `{ty}`, which is invalid") + } else { + format!("attempted to leave type `{ty}` uninitialized, which is invalid") + } + }) + }); + let msg = bx.const_str(&msg_str); + + // Obtain the panic entry point. + let (fn_abi, llfn, instance) = + common::build_langcall(bx, Some(source_info.span), LangItem::PanicNounwind); + + // Codegen the actual panic invoke/call. + Some(helper.do_call( + self, + bx, + fn_abi, + llfn, + &[msg.0, msg.1], + target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), + unwind, + &[], + Some(instance), + mergeable_succ, + )) } fn codegen_call_terminator( @@ -903,42 +901,127 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_span: Span, mergeable_succ: bool, ) -> MergingSucc { - let source_info = terminator.source_info; - let span = source_info.span; + let source_info = mir::SourceInfo { span: fn_span, ..terminator.source_info }; // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. let callee = self.codegen_operand(bx, func); let (instance, mut llfn) = match *callee.layout.ty.kind() { - ty::FnDef(def_id, args) => ( - Some(ty::Instance::expect_resolve( + ty::FnDef(def_id, generic_args) => { + let instance = ty::Instance::expect_resolve( bx.tcx(), bx.typing_env(), def_id, - args, + generic_args, fn_span, - )), - None, - ), + ); + + let instance = match instance.def { + // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, + // it is `func returning noop future` + ty::InstanceKind::DropGlue(_, None) => { + // Empty drop glue; a no-op. + let target = target.unwrap(); + return helper.funclet_br(self, bx, target, mergeable_succ); + } + ty::InstanceKind::Intrinsic(def_id) => { + let intrinsic = bx.tcx().intrinsic(def_id).unwrap(); + if let Some(merging_succ) = self.codegen_panic_intrinsic( + &helper, + bx, + intrinsic, + instance, + source_info, + target, + unwind, + mergeable_succ, + ) { + return merging_succ; + } + + let result_layout = + self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + + let (result, store_in_local) = if result_layout.is_zst() { + ( + PlaceRef::new_sized(bx.const_undef(bx.type_ptr()), result_layout), + None, + ) + } else if let Some(local) = destination.as_local() { + match self.locals[local] { + LocalRef::Place(dest) => (dest, None), + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::PendingOperand => { + // Currently, intrinsics always need a location to store + // the result, so we create a temporary `alloca` for the + // result. + let tmp = PlaceRef::alloca(bx, result_layout); + tmp.storage_live(bx); + (tmp, Some(local)) + } + LocalRef::Operand(_) => { + bug!("place local already assigned to"); + } + } + } else { + (self.codegen_place(bx, destination.as_ref()), None) + }; + + if result.val.align < result.layout.align.abi { + // Currently, MIR code generation does not create calls + // that store directly to fields of packed structs (in + // fact, the calls it creates write only to temps). + // + // If someone changes that, please update this code path + // to create a temporary. + span_bug!(self.mir.span, "can't directly store to unaligned value"); + } + + let args: Vec<_> = + args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect(); + + match self.codegen_intrinsic_call(bx, instance, &args, result, source_info) + { + Ok(()) => { + if let Some(local) = store_in_local { + let op = bx.load_operand(result); + result.storage_dead(bx); + self.overwrite_local(local, LocalRef::Operand(op)); + self.debug_introduce_local(bx, local); + } + + return if let Some(target) = target { + helper.funclet_br(self, bx, target, mergeable_succ) + } else { + bx.unreachable(); + MergingSucc::False + }; + } + Err(instance) => { + if intrinsic.must_be_overridden { + span_bug!( + fn_span, + "intrinsic {} must be overridden by codegen backend, but isn't", + intrinsic.name, + ); + } + instance + } + } + } + _ => instance, + }; + + (Some(instance), None) + } ty::FnPtr(..) => (None, Some(callee.immediate())), _ => bug!("{} is not callable", callee.layout.ty), }; - let def = instance.map(|i| i.def); - - // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, - // it is `func returning noop future` - if let Some(ty::InstanceKind::DropGlue(_, None)) = def { - // Empty drop glue; a no-op. - let target = target.unwrap(); - return helper.funclet_br(self, bx, target, mergeable_succ); - } - // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. let sig = callee.layout.ty.fn_sig(bx.tcx()); - let abi = sig.abi(); let extra_args = &args[sig.inputs().skip_binder().len()..]; let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| { @@ -954,93 +1037,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // The arguments we'll be passing. Plus one to account for outptr, if used. let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize; - let instance = match def { - Some(ty::InstanceKind::Intrinsic(def_id)) => { - let intrinsic = bx.tcx().intrinsic(def_id).unwrap(); - if let Some(merging_succ) = self.codegen_panic_intrinsic( - &helper, - bx, - intrinsic, - instance, - source_info, - target, - unwind, - mergeable_succ, - ) { - return merging_succ; - } - - let mut llargs = Vec::with_capacity(1); - let ret_dest = self.make_return_dest( - bx, - destination, - &fn_abi.ret, - &mut llargs, - Some(intrinsic), - ); - let dest = match ret_dest { - _ if fn_abi.ret.is_indirect() => llargs[0], - ReturnDest::Nothing => bx.const_undef(bx.type_ptr()), - ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.val.llval, - ReturnDest::DirectOperand(_) => { - bug!("Cannot use direct operand with an intrinsic call") - } - }; - - let args: Vec<_> = - args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect(); - - if matches!(intrinsic, ty::IntrinsicDef { name: sym::caller_location, .. }) { - let location = self - .get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info }); - - assert_eq!(llargs, []); - if let ReturnDest::IndirectOperand(tmp, _) = ret_dest { - location.val.store(bx, tmp); - } - self.store_return(bx, ret_dest, &fn_abi.ret, location.immediate()); - return helper.funclet_br(self, bx, target.unwrap(), mergeable_succ); - } - - let instance = *instance.as_ref().unwrap(); - match Self::codegen_intrinsic_call(bx, instance, fn_abi, &args, dest, span) { - Ok(()) => { - if let ReturnDest::IndirectOperand(dst, _) = ret_dest { - self.store_return(bx, ret_dest, &fn_abi.ret, dst.val.llval); - } - - return if let Some(target) = target { - helper.funclet_br(self, bx, target, mergeable_succ) - } else { - bx.unreachable(); - MergingSucc::False - }; - } - Err(instance) => { - if intrinsic.must_be_overridden { - span_bug!( - span, - "intrinsic {} must be overridden by codegen backend, but isn't", - intrinsic.name, - ); - } - Some(instance) - } - } - } - _ => instance, - }; - let mut llargs = Vec::with_capacity(arg_count); // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, None); + let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); let destination = target.map(|target| (return_dest, target)); // Split the rust-call tupled arguments off. - let (first_args, untuple) = if abi == ExternAbi::RustCall + let (first_args, untuple) = if sig.abi() == ExternAbi::RustCall && let Some((tup, args)) = args.split_last() { (args, Some(tup)) @@ -1055,7 +1061,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { 'make_args: for (i, arg) in first_args.iter().enumerate() { let mut op = self.codegen_operand(bx, &arg.node); - if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, def) { + if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, instance.map(|i| i.def)) { match op.val { Pair(data_ptr, meta) => { // In the case of Rc<Self>, we need to explicitly pass a @@ -1109,7 +1115,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Make sure that we've actually unwrapped the rcvr down // to a pointer or ref to `dyn* Trait`. if !op.layout.ty.builtin_deref(true).unwrap().is_dyn_star() { - span_bug!(span, "can't codegen a virtual call on {:#?}", op); + span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op); } let place = op.deref(bx.cx()); let data_place = place.project_field(bx, 0); @@ -1125,7 +1131,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { continue; } _ => { - span_bug!(span, "can't codegen a virtual call on {:#?}", op); + span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op); } } } @@ -1175,8 +1181,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir_args + 1, "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR: {instance:?} {fn_span:?} {fn_abi:?}", ); - let location = - self.get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info }); + let location = self.get_caller_location(bx, source_info); debug!( "codegen_call_terminator({:?}): location={:?} (fn_span {:?})", terminator, location, fn_span @@ -1195,9 +1200,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let fn_ptr = match (instance, llfn) { (Some(instance), None) => bx.get_fn_addr(instance), (_, Some(llfn)) => llfn, - _ => span_bug!(span, "no instance or llfn for call"), + _ => span_bug!(fn_span, "no instance or llfn for call"), }; - self.set_debug_loc(bx, mir::SourceInfo { span: fn_span, ..source_info }); + self.set_debug_loc(bx, source_info); helper.do_call( self, bx, @@ -1667,7 +1672,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { tuple.layout.fields.count() } - fn get_caller_location( + pub(super) fn get_caller_location( &mut self, bx: &mut Bx, source_info: mir::SourceInfo, @@ -1868,7 +1873,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { dest: mir::Place<'tcx>, fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, llargs: &mut Vec<Bx::Value>, - intrinsic: Option<ty::IntrinsicDef>, ) -> ReturnDest<'tcx, Bx::Value> { // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { @@ -1888,13 +1892,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { tmp.storage_live(bx); llargs.push(tmp.val.llval); ReturnDest::IndirectOperand(tmp, index) - } else if intrinsic.is_some() { - // Currently, intrinsics always need a location to store - // the result, so we create a temporary `alloca` for the - // result. - let tmp = PlaceRef::alloca(bx, fn_ret.layout); - tmp.storage_live(bx); - ReturnDest::IndirectOperand(tmp, index) } else { ReturnDest::DirectOperand(index) }; @@ -1904,7 +1901,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } } else { - self.codegen_place(bx, mir::PlaceRef { local: dest.local, projection: dest.projection }) + self.codegen_place(bx, dest.as_ref()) }; if fn_ret.is_indirect() { if dest.val.align < dest.layout.align.abi { diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index b0fcfee2adf..a6d159c51e1 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,9 +1,9 @@ use rustc_abi::WrappingRange; +use rustc_middle::mir::SourceInfo; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; -use rustc_span::{Span, sym}; -use rustc_target::callconv::{FnAbi, PassMode}; +use rustc_span::sym; use super::FunctionCx; use super::operand::OperandRef; @@ -52,13 +52,14 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// In the `Err` case, returns the instance that should be called instead. pub fn codegen_intrinsic_call( + &mut self, bx: &mut Bx, instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, Bx::Value>], - llresult: Bx::Value, - span: Span, + result: PlaceRef<'tcx, Bx::Value>, + source_info: SourceInfo, ) -> Result<(), ty::Instance<'tcx>> { + let span = source_info.span; let callee_ty = instance.ty(bx.tcx(), bx.typing_env()); let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { @@ -97,7 +98,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); - let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); let llval = match name { sym::abort => { @@ -105,6 +105,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return Ok(()); } + sym::caller_location => { + let location = self.get_caller_location(bx, source_info); + location.val.store(bx, result); + return Ok(()); + } + sym::va_start => bx.va_start(args[0].immediate()), sym::va_end => bx.va_end(args[0].immediate()), sym::size_of_val => { @@ -528,18 +534,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => { // Need to use backend-specific things in the implementation. - return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); + return bx.codegen_intrinsic_call(instance, args, result, span); } }; - if !fn_abi.ret.is_ignore() { - if let PassMode::Cast { .. } = &fn_abi.ret.mode { - bx.store_to_place(llval, result.val); - } else { - OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout) - .val - .store(bx, result); - } + if result.layout.ty.is_bool() { + OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout) + .val + .store(bx, result); + } else if !result.layout.ty.is_unit() { + bx.store_to_place(llval, result.val); } Ok(()) } diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 88cf8dbf0c5..a07c569a032 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -1,9 +1,9 @@ -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty; use rustc_span::Span; -use rustc_target::callconv::FnAbi; use super::BackendTypes; use crate::mir::operand::OperandRef; +use crate::mir::place::PlaceRef; pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { /// Remember to add all intrinsics here, in `compiler/rustc_hir_analysis/src/check/mod.rs`, @@ -14,9 +14,8 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { fn codegen_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, Self::Value>], - llresult: Self::Value, + result: PlaceRef<'tcx, Self::Value>, span: Span, ) -> Result<(), ty::Instance<'tcx>>; diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 81d4d59ee04..e5749ba96a6 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -1319,10 +1319,10 @@ impl InvocationCollectorNode for P<ast::Item> { let mut idents = Vec::new(); collect_use_tree_leaves(ut, &mut idents); - return idents; + idents + } else { + self.kind.ident().into_iter().collect() } - - if let Some(ident) = self.kind.ident() { vec![ident] } else { vec![] } } } diff --git a/compiler/rustc_mir_build/src/builder/scope.rs b/compiler/rustc_mir_build/src/builder/scope.rs index 2a30777e98c..67988f1fcbc 100644 --- a/compiler/rustc_mir_build/src/builder/scope.rs +++ b/compiler/rustc_mir_build/src/builder/scope.rs @@ -209,7 +209,7 @@ const ROOT_NODE: DropIdx = DropIdx::ZERO; #[derive(Debug)] struct DropTree { /// Nodes in the drop tree, containing drop data and a link to the next node. - drops: IndexVec<DropIdx, DropNode>, + drop_nodes: IndexVec<DropIdx, DropNode>, /// Map for finding the index of an existing node, given its contents. existing_drops_map: FxHashMap<DropNodeKey, DropIdx>, /// Edges into the `DropTree` that need to be added once it's lowered. @@ -230,7 +230,6 @@ struct DropNode { struct DropNodeKey { next: DropIdx, local: Local, - kind: DropKind, } impl Scope { @@ -278,8 +277,8 @@ impl DropTree { let fake_source_info = SourceInfo::outermost(DUMMY_SP); let fake_data = DropData { source_info: fake_source_info, local: Local::MAX, kind: DropKind::Storage }; - let drops = IndexVec::from_raw(vec![DropNode { data: fake_data, next: DropIdx::MAX }]); - Self { drops, entry_points: Vec::new(), existing_drops_map: FxHashMap::default() } + let drop_nodes = IndexVec::from_raw(vec![DropNode { data: fake_data, next: DropIdx::MAX }]); + Self { drop_nodes, entry_points: Vec::new(), existing_drops_map: FxHashMap::default() } } /// Adds a node to the drop tree, consisting of drop data and the index of @@ -288,12 +287,12 @@ impl DropTree { /// If there is already an equivalent node in the tree, nothing is added, and /// that node's index is returned. Otherwise, the new node's index is returned. fn add_drop(&mut self, data: DropData, next: DropIdx) -> DropIdx { - let drops = &mut self.drops; + let drop_nodes = &mut self.drop_nodes; *self .existing_drops_map - .entry(DropNodeKey { next, local: data.local, kind: data.kind }) + .entry(DropNodeKey { next, local: data.local }) // Create a new node, and also add its index to the map. - .or_insert_with(|| drops.push(DropNode { data, next })) + .or_insert_with(|| drop_nodes.push(DropNode { data, next })) } /// Registers `from` as an entry point to this drop tree, at `to`. @@ -301,7 +300,7 @@ impl DropTree { /// During [`Self::build_mir`], `from` will be linked to the corresponding /// block within the drop tree. fn add_entry_point(&mut self, from: BasicBlock, to: DropIdx) { - debug_assert!(to < self.drops.next_index()); + debug_assert!(to < self.drop_nodes.next_index()); self.entry_points.push((to, from)); } @@ -341,10 +340,10 @@ impl DropTree { Own, } - let mut blocks = IndexVec::from_elem(None, &self.drops); + let mut blocks = IndexVec::from_elem(None, &self.drop_nodes); blocks[ROOT_NODE] = root_node; - let mut needs_block = IndexVec::from_elem(Block::None, &self.drops); + let mut needs_block = IndexVec::from_elem(Block::None, &self.drop_nodes); if root_node.is_some() { // In some cases (such as drops for `continue`) the root node // already has a block. In this case, make sure that we don't @@ -356,7 +355,7 @@ impl DropTree { let entry_points = &mut self.entry_points; entry_points.sort(); - for (drop_idx, drop_node) in self.drops.iter_enumerated().rev() { + for (drop_idx, drop_node) in self.drop_nodes.iter_enumerated().rev() { if entry_points.last().is_some_and(|entry_point| entry_point.0 == drop_idx) { let block = *blocks[drop_idx].get_or_insert_with(|| T::make_block(cfg)); needs_block[drop_idx] = Block::Own; @@ -396,7 +395,7 @@ impl DropTree { cfg: &mut CFG<'tcx>, blocks: &IndexSlice<DropIdx, Option<BasicBlock>>, ) { - for (drop_idx, drop_node) in self.drops.iter_enumerated().rev() { + for (drop_idx, drop_node) in self.drop_nodes.iter_enumerated().rev() { let Some(block) = blocks[drop_idx] else { continue }; match drop_node.data.kind { DropKind::Value => { @@ -726,11 +725,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { drops }; - let drop_idx = self.scopes.scopes[scope_index + 1..] - .iter() - .flat_map(|scope| &scope.drops) - .fold(ROOT_NODE, |drop_idx, &drop| drops.add_drop(drop, drop_idx)); - + let mut drop_idx = ROOT_NODE; + for scope in &self.scopes.scopes[scope_index + 1..] { + for drop in &scope.drops { + drop_idx = drops.add_drop(*drop, drop_idx); + } + } drops.add_entry_point(block, drop_idx); // `build_drop_trees` doesn't have access to our source_info, so we @@ -829,9 +829,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // `unwind_to` should drop the value that we're about to // schedule. If dropping this value panics, then we continue // with the *next* value on the unwind path. - debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local); - debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind); - unwind_to = unwind_drops.drops[unwind_to].next; + debug_assert_eq!( + unwind_drops.drop_nodes[unwind_to].data.local, + drop_data.local + ); + debug_assert_eq!( + unwind_drops.drop_nodes[unwind_to].data.kind, + drop_data.kind + ); + unwind_to = unwind_drops.drop_nodes[unwind_to].next; let mut unwind_entry_point = unwind_to; @@ -1551,14 +1557,14 @@ where // // We adjust this BEFORE we create the drop (e.g., `drops[n]`) // because `drops[n]` should unwind to `drops[n-1]`. - debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local); - debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind); - unwind_to = unwind_drops.drops[unwind_to].next; + debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.local, drop_data.local); + debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind); + unwind_to = unwind_drops.drop_nodes[unwind_to].next; if let Some(idx) = dropline_to { - debug_assert_eq!(coroutine_drops.drops[idx].data.local, drop_data.local); - debug_assert_eq!(coroutine_drops.drops[idx].data.kind, drop_data.kind); - dropline_to = Some(coroutine_drops.drops[idx].next); + debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.local, drop_data.local); + debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.kind, drop_data.kind); + dropline_to = Some(coroutine_drops.drop_nodes[idx].next); } // If the operand has been moved, and we are not on an unwind @@ -1598,9 +1604,12 @@ where // cases we emit things ALSO on the unwind path, so we need to adjust // `unwind_to` in that case. if storage_dead_on_unwind { - debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local); - debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind); - unwind_to = unwind_drops.drops[unwind_to].next; + debug_assert_eq!( + unwind_drops.drop_nodes[unwind_to].data.local, + drop_data.local + ); + debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind); + unwind_to = unwind_drops.drop_nodes[unwind_to].next; } // If the operand has been moved, and we are not on an unwind @@ -1629,14 +1638,17 @@ where // the storage-dead has completed, we need to adjust the `unwind_to` pointer // so that any future drops we emit will not register storage-dead. if storage_dead_on_unwind { - debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local); - debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind); - unwind_to = unwind_drops.drops[unwind_to].next; + debug_assert_eq!( + unwind_drops.drop_nodes[unwind_to].data.local, + drop_data.local + ); + debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind); + unwind_to = unwind_drops.drop_nodes[unwind_to].next; } if let Some(idx) = dropline_to { - debug_assert_eq!(coroutine_drops.drops[idx].data.local, drop_data.local); - debug_assert_eq!(coroutine_drops.drops[idx].data.kind, drop_data.kind); - dropline_to = Some(coroutine_drops.drops[idx].next); + debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.local, drop_data.local); + debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.kind, drop_data.kind); + dropline_to = Some(coroutine_drops.drop_nodes[idx].next); } // Only temps and vars need their storage dead. assert!(local.index() > arg_count); @@ -1663,10 +1675,10 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> { let is_coroutine = self.coroutine.is_some(); // Link the exit drop tree to unwind drop tree. - if drops.drops.iter().any(|drop_node| drop_node.data.kind == DropKind::Value) { + if drops.drop_nodes.iter().any(|drop_node| drop_node.data.kind == DropKind::Value) { let unwind_target = self.diverge_cleanup_target(else_scope, span); let mut unwind_indices = IndexVec::from_elem_n(unwind_target, 1); - for (drop_idx, drop_node) in drops.drops.iter_enumerated().skip(1) { + for (drop_idx, drop_node) in drops.drop_nodes.iter_enumerated().skip(1) { match drop_node.data.kind { DropKind::Storage | DropKind::ForLint => { if is_coroutine { @@ -1695,35 +1707,29 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> { } // Link the exit drop tree to dropline drop tree (coroutine drop path) for async drops if is_coroutine - && drops.drops.iter().any(|DropNode { data, next: _ }| { + && drops.drop_nodes.iter().any(|DropNode { data, next: _ }| { data.kind == DropKind::Value && self.is_async_drop(data.local) }) { let dropline_target = self.diverge_dropline_target(else_scope, span); let mut dropline_indices = IndexVec::from_elem_n(dropline_target, 1); - for (drop_idx, drop_data) in drops.drops.iter_enumerated().skip(1) { + for (drop_idx, drop_data) in drops.drop_nodes.iter_enumerated().skip(1) { + let coroutine_drop = self + .scopes + .coroutine_drops + .add_drop(drop_data.data, dropline_indices[drop_data.next]); match drop_data.data.kind { - DropKind::Storage | DropKind::ForLint => { - let coroutine_drop = self - .scopes - .coroutine_drops - .add_drop(drop_data.data, dropline_indices[drop_data.next]); - dropline_indices.push(coroutine_drop); - } + DropKind::Storage | DropKind::ForLint => {} DropKind::Value => { - let coroutine_drop = self - .scopes - .coroutine_drops - .add_drop(drop_data.data, dropline_indices[drop_data.next]); if self.is_async_drop(drop_data.data.local) { self.scopes.coroutine_drops.add_entry_point( blocks[drop_idx].unwrap(), dropline_indices[drop_data.next], ); } - dropline_indices.push(coroutine_drop); } } + dropline_indices.push(coroutine_drop); } } blocks[ROOT_NODE].map(BasicBlock::unit) @@ -1769,11 +1775,11 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> { // prevent drop elaboration from creating drop flags that would have // to be captured by the coroutine. I'm not sure how important this // optimization is, but it is here. - for (drop_idx, drop_node) in drops.drops.iter_enumerated() { + for (drop_idx, drop_node) in drops.drop_nodes.iter_enumerated() { if let DropKind::Value = drop_node.data.kind && let Some(bb) = blocks[drop_idx] { - debug_assert!(drop_node.next < drops.drops.next_index()); + debug_assert!(drop_node.next < drops.drop_nodes.next_index()); drops.entry_points.push((drop_node.next, bb)); } } diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 396ded96bde..ccc3410674b 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -515,8 +515,8 @@ impl<'a> Parser<'a> { fn error_block_no_opening_brace_msg(&mut self, msg: Cow<'static, str>) -> Diag<'a> { let prev = self.prev_token.span; let sp = self.token.span; - let mut e = self.dcx().struct_span_err(sp, msg); - self.label_expected_raw_ref(&mut e); + let mut err = self.dcx().struct_span_err(sp, msg); + self.label_expected_raw_ref(&mut err); let do_not_suggest_help = self.token.is_keyword(kw::In) || self.token == token::Colon @@ -558,20 +558,19 @@ impl<'a> Parser<'a> { stmt.span }; self.suggest_fixes_misparsed_for_loop_head( - &mut e, + &mut err, prev.between(sp), stmt_span, &stmt.kind, ); } Err(e) => { - self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore); - e.cancel(); + e.delay_as_bug(); } _ => {} } - e.span_label(sp, "expected `{`"); - e + err.span_label(sp, "expected `{`"); + err } fn suggest_fixes_misparsed_for_loop_head( diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 1d024694049..df8b5a6b181 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -255,7 +255,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { self.check_generic_attr(hir_id, attr, target, Target::Fn); self.check_proc_macro(hir_id, target, ProcMacroKind::Derive) } - [sym::autodiff, ..] => { + [sym::autodiff_forward, ..] | [sym::autodiff_reverse, ..] => { self.check_autodiff(hir_id, attr, span, target) } [sym::coroutine, ..] => { diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs index d09750fa281..201b1c0a493 100644 --- a/compiler/rustc_resolve/src/diagnostics.rs +++ b/compiler/rustc_resolve/src/diagnostics.rs @@ -6,7 +6,7 @@ use rustc_ast::{ }; use rustc_ast_pretty::pprust; use rustc_attr_data_structures::{self as attr, Stability}; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::unord::UnordSet; use rustc_errors::codes::*; use rustc_errors::{ @@ -2623,7 +2623,53 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> { }; for &StrippedCfgItem { parent_module, ident, ref cfg } in symbols { - if parent_module != module || ident.name != *segment { + if ident.name != *segment { + continue; + } + + fn comes_from_same_module_for_glob( + r: &Resolver<'_, '_>, + parent_module: DefId, + module: DefId, + visited: &mut FxHashMap<DefId, bool>, + ) -> bool { + if let Some(&cached) = visited.get(&parent_module) { + // this branch is prevent from being called recursively infinity, + // because there has some cycles in globs imports, + // see more spec case at `tests/ui/cfg/diagnostics-reexport-2.rs#reexport32` + return cached; + } + visited.insert(parent_module, false); + let res = r.module_map.get(&parent_module).is_some_and(|m| { + for importer in m.glob_importers.borrow().iter() { + if let Some(next_parent_module) = importer.parent_scope.module.opt_def_id() + { + if next_parent_module == module + || comes_from_same_module_for_glob( + r, + next_parent_module, + module, + visited, + ) + { + return true; + } + } + } + false + }); + visited.insert(parent_module, res); + res + } + + let comes_from_same_module = parent_module == module + || comes_from_same_module_for_glob( + self, + parent_module, + module, + &mut Default::default(), + ); + if !comes_from_same_module { continue; } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 0447713fb05..8f7e72f0ae1 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -244,6 +244,7 @@ symbols! { FnMut, FnOnce, Formatter, + Forward, From, FromIterator, FromResidual, @@ -339,6 +340,7 @@ symbols! { Result, ResumeTy, Return, + Reverse, Right, Rust, RustaceansAreAwesome, @@ -522,7 +524,8 @@ symbols! { audit_that, augmented_assignments, auto_traits, - autodiff, + autodiff_forward, + autodiff_reverse, automatically_derived, avx, avx10_target_feature, |