diff options
Diffstat (limited to 'compiler')
116 files changed, 3154 insertions, 1389 deletions
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 571aaf631bd..b30ff058a30 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1300,12 +1300,18 @@ impl Abi { matches!(*self, Abi::Uninhabited) } - /// Returns `true` is this is a scalar type + /// Returns `true` if this is a scalar type #[inline] pub fn is_scalar(&self) -> bool { matches!(*self, Abi::Scalar(_)) } + /// Returns `true` if this is a bool + #[inline] + pub fn is_bool(&self) -> bool { + matches!(*self, Abi::Scalar(s) if s.is_bool()) + } + /// Returns the fixed alignment of this ABI, if any is mandated. pub fn inherent_align<C: HasDataLayout>(&self, cx: &C) -> Option<AbiAndPrefAlign> { Some(match *self { @@ -1348,6 +1354,23 @@ impl Abi { Abi::Uninhabited | Abi::Aggregate { .. } => Abi::Aggregate { sized: true }, } } + + pub fn eq_up_to_validity(&self, other: &Self) -> bool { + match (self, other) { + // Scalar, Vector, ScalarPair have `Scalar` in them where we ignore validity ranges. + // We do *not* ignore the sign since it matters for some ABIs (e.g. s390x). + (Abi::Scalar(l), Abi::Scalar(r)) => l.primitive() == r.primitive(), + ( + Abi::Vector { element: element_l, count: count_l }, + Abi::Vector { element: element_r, count: count_r }, + ) => element_l.primitive() == element_r.primitive() && count_l == count_r, + (Abi::ScalarPair(l1, l2), Abi::ScalarPair(r1, r2)) => { + l1.primitive() == r1.primitive() && l2.primitive() == r2.primitive() + } + // Everything else must be strictly identical. + _ => self == other, + } + } } #[derive(PartialEq, Eq, Hash, Clone, Debug)] @@ -1686,6 +1709,22 @@ impl LayoutS { Abi::Aggregate { sized } => sized && self.size.bytes() == 0, } } + + /// Checks if these two `Layout` are equal enough to be considered "the same for all function + /// call ABIs". Note however that real ABIs depend on more details that are not reflected in the + /// `Layout`; the `PassMode` need to be compared as well. + pub fn eq_abi(&self, other: &Self) -> bool { + // The one thing that we are not capturing here is that for unsized types, the metadata must + // also have the same ABI, and moreover that the same metadata leads to the same size. The + // 2nd point is quite hard to check though. + self.size == other.size + && self.is_sized() == other.is_sized() + && self.abi.eq_up_to_validity(&other.abi) + && self.abi.is_bool() == other.abi.is_bool() + && self.align.abi == other.align.abi + && self.max_repr_align == other.max_repr_align + && self.unadjusted_abi_align == other.unadjusted_abi_align + } } #[derive(Copy, Clone, Debug)] diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs index ce847906fb9..eff362f3ff0 100644 --- a/compiler/rustc_ast_lowering/src/index.rs +++ b/compiler/rustc_ast_lowering/src/index.rs @@ -2,19 +2,17 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sorted_map::SortedMap; use rustc_hir as hir; use rustc_hir::def_id::LocalDefId; -use rustc_hir::definitions; use rustc_hir::intravisit::{self, Visitor}; use rustc_hir::*; use rustc_index::{Idx, IndexVec}; use rustc_middle::span_bug; -use rustc_session::Session; -use rustc_span::source_map::SourceMap; +use rustc_middle::ty::TyCtxt; use rustc_span::{Span, DUMMY_SP}; /// A visitor that walks over the HIR and collects `Node`s into a HIR map. pub(super) struct NodeCollector<'a, 'hir> { - /// Source map - source_map: &'a SourceMap, + tcx: TyCtxt<'hir>, + bodies: &'a SortedMap<ItemLocalId, &'hir Body<'hir>>, /// Outputs @@ -25,14 +23,11 @@ pub(super) struct NodeCollector<'a, 'hir> { parent_node: hir::ItemLocalId, owner: OwnerId, - - definitions: &'a definitions::Definitions, } -#[instrument(level = "debug", skip(sess, definitions, bodies))] +#[instrument(level = "debug", skip(tcx, bodies))] pub(super) fn index_hir<'hir>( - sess: &Session, - definitions: &definitions::Definitions, + tcx: TyCtxt<'hir>, item: hir::OwnerNode<'hir>, bodies: &SortedMap<ItemLocalId, &'hir Body<'hir>>, ) -> (IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>, FxHashMap<LocalDefId, ItemLocalId>) { @@ -42,8 +37,7 @@ pub(super) fn index_hir<'hir>( // used. nodes.push(Some(ParentedNode { parent: ItemLocalId::INVALID, node: item.into() })); let mut collector = NodeCollector { - source_map: sess.source_map(), - definitions, + tcx, owner: item.def_id(), parent_node: ItemLocalId::new(0), nodes, @@ -79,11 +73,17 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { span, "inconsistent HirId at `{:?}` for `{:?}`: \ current_dep_node_owner={} ({:?}), hir_id.owner={} ({:?})", - self.source_map.span_to_diagnostic_string(span), + self.tcx.sess.source_map().span_to_diagnostic_string(span), node, - self.definitions.def_path(self.owner.def_id).to_string_no_crate_verbose(), + self.tcx + .definitions_untracked() + .def_path(self.owner.def_id) + .to_string_no_crate_verbose(), self.owner, - self.definitions.def_path(hir_id.owner.def_id).to_string_no_crate_verbose(), + self.tcx + .definitions_untracked() + .def_path(hir_id.owner.def_id) + .to_string_no_crate_verbose(), hir_id.owner, ) } diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index a6d1ef33f40..41db61d391a 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -671,8 +671,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } else { (None, None) }; - let (nodes, parenting) = - index::index_hir(self.tcx.sess, &*self.tcx.definitions_untracked(), node, &bodies); + let (nodes, parenting) = index::index_hir(self.tcx, node, &bodies); let nodes = hir::OwnerNodes { opt_hash_including_bodies, nodes, bodies }; let attrs = hir::AttributeMap { map: attrs, opt_hash: attrs_hash }; diff --git a/compiler/rustc_borrowck/messages.ftl b/compiler/rustc_borrowck/messages.ftl index 67fdb671742..d8ca62565fa 100644 --- a/compiler/rustc_borrowck/messages.ftl +++ b/compiler/rustc_borrowck/messages.ftl @@ -166,6 +166,8 @@ borrowck_returned_lifetime_wrong = borrowck_returned_ref_escaped = returns a reference to a captured variable which escapes the closure body +borrowck_simd_shuffle_last_const = last argument of `simd_shuffle` is required to be a `const` item + borrowck_suggest_create_freash_reborrow = consider reborrowing the `Pin` instead of moving it diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs index fe4a45b3898..48d09f2c2b2 100644 --- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs @@ -2130,21 +2130,27 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { /// misleading users in cases like `tests/ui/nll/borrowed-temporary-error.rs`. /// We could expand the analysis to suggest hoising all of the relevant parts of /// the users' code to make the code compile, but that could be too much. - struct NestedStatementVisitor { + /// We found the `prop_expr` by the way to check whether the expression is a `FormatArguments`, + /// which is a special case since it's generated by the compiler. + struct NestedStatementVisitor<'tcx> { span: Span, current: usize, found: usize, + prop_expr: Option<&'tcx hir::Expr<'tcx>>, } - impl<'tcx> Visitor<'tcx> for NestedStatementVisitor { - fn visit_block(&mut self, block: &hir::Block<'tcx>) { + impl<'tcx> Visitor<'tcx> for NestedStatementVisitor<'tcx> { + fn visit_block(&mut self, block: &'tcx hir::Block<'tcx>) { self.current += 1; walk_block(self, block); self.current -= 1; } - fn visit_expr(&mut self, expr: &hir::Expr<'tcx>) { + fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) { if self.span == expr.span.source_callsite() { self.found = self.current; + if self.prop_expr.is_none() { + self.prop_expr = Some(expr); + } } walk_expr(self, expr); } @@ -2162,22 +2168,40 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> { span: proper_span, current: 0, found: 0, + prop_expr: None, }; visitor.visit_stmt(stmt); + + let typeck_results = self.infcx.tcx.typeck(self.mir_def_id()); + let expr_ty: Option<Ty<'_>> = visitor.prop_expr.map(|expr| typeck_results.expr_ty(expr).peel_refs()); + + let is_format_arguments_item = + if let Some(expr_ty) = expr_ty + && let ty::Adt(adt, _) = expr_ty.kind() { + self.infcx.tcx.lang_items().get(LangItem::FormatArguments) == Some(adt.did()) + } else { + false + }; + if visitor.found == 0 && stmt.span.contains(proper_span) && let Some(p) = sm.span_to_margin(stmt.span) && let Ok(s) = sm.span_to_snippet(proper_span) { - let addition = format!("let binding = {};\n{}", s, " ".repeat(p)); - err.multipart_suggestion_verbose( - msg, - vec![ - (stmt.span.shrink_to_lo(), addition), - (proper_span, "binding".to_string()), - ], - Applicability::MaybeIncorrect, - ); + if !is_format_arguments_item { + let addition = format!("let binding = {};\n{}", s, " ".repeat(p)); + err.multipart_suggestion_verbose( + msg, + vec![ + (stmt.span.shrink_to_lo(), addition), + (proper_span, "binding".to_string()), + ], + Applicability::MaybeIncorrect, + ); + } else { + err.note("the result of `format_args!` can only be assigned directly if no placeholders in it's arguments are used"); + err.note("to learn more, visit <https://doc.rust-lang.org/std/macro.format_args.html>"); + } suggested = true; break; } diff --git a/compiler/rustc_borrowck/src/session_diagnostics.rs b/compiler/rustc_borrowck/src/session_diagnostics.rs index d1d8cfa74aa..ca3ccf439f2 100644 --- a/compiler/rustc_borrowck/src/session_diagnostics.rs +++ b/compiler/rustc_borrowck/src/session_diagnostics.rs @@ -452,3 +452,10 @@ pub(crate) enum TypeNoCopy<'a, 'tcx> { #[note(borrowck_ty_no_impl_copy)] Note { is_partial_move: bool, ty: Ty<'tcx>, place: &'a str }, } + +#[derive(Diagnostic)] +#[diag(borrowck_simd_shuffle_last_const)] +pub(crate) struct SimdShuffleLastConst { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 28286243e82..0f661421cdc 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -50,7 +50,7 @@ use rustc_mir_dataflow::impls::MaybeInitializedPlaces; use rustc_mir_dataflow::move_paths::MoveData; use rustc_mir_dataflow::ResultsCursor; -use crate::session_diagnostics::MoveUnsized; +use crate::session_diagnostics::{MoveUnsized, SimdShuffleLastConst}; use crate::{ borrow_set::BorrowSet, constraints::{OutlivesConstraint, OutlivesConstraintSet}, @@ -1426,7 +1426,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { .add_element(region_vid, term_location); } - self.check_call_inputs(body, term, &sig, args, term_location, *call_source); + self.check_call_inputs(body, term, func, &sig, args, term_location, *call_source); } TerminatorKind::Assert { cond, msg, .. } => { self.check_operand(cond, term_location); @@ -1546,25 +1546,36 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } + #[instrument(level = "debug", skip(self, body, term, func, term_location, call_source))] fn check_call_inputs( &mut self, body: &Body<'tcx>, term: &Terminator<'tcx>, + func: &Operand<'tcx>, sig: &ty::FnSig<'tcx>, args: &[Operand<'tcx>], term_location: Location, call_source: CallSource, ) { - debug!("check_call_inputs({:?}, {:?})", sig, args); if args.len() < sig.inputs().len() || (args.len() > sig.inputs().len() && !sig.c_variadic) { span_mirbug!(self, term, "call to {:?} with wrong # of args", sig); } - let func_ty = if let TerminatorKind::Call { func, .. } = &term.kind { - Some(func.ty(body, self.infcx.tcx)) - } else { - None - }; + let func_ty = func.ty(body, self.infcx.tcx); + if let ty::FnDef(def_id, _) = *func_ty.kind() { + if self.tcx().is_intrinsic(def_id) { + match self.tcx().item_name(def_id) { + sym::simd_shuffle => { + if !matches!(args[2], Operand::Constant(_)) { + self.tcx() + .sess + .emit_err(SimdShuffleLastConst { span: term.source_info.span }); + } + } + _ => {} + } + } + } debug!(?func_ty); for (n, (fn_arg, op_arg)) in iter::zip(sig.inputs(), args).enumerate() { @@ -1572,7 +1583,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let op_arg_ty = self.normalize(op_arg_ty, term_location); let category = if call_source.from_hir_call() { - ConstraintCategory::CallArgument(self.infcx.tcx.erase_regions(func_ty)) + ConstraintCategory::CallArgument(Some(self.infcx.tcx.erase_regions(func_ty))) } else { ConstraintCategory::Boring }; diff --git a/compiler/rustc_codegen_cranelift/docs/usage.md b/compiler/rustc_codegen_cranelift/docs/usage.md index c6210f958d6..135a51ce392 100644 --- a/compiler/rustc_codegen_cranelift/docs/usage.md +++ b/compiler/rustc_codegen_cranelift/docs/usage.md @@ -54,7 +54,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic + echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic } function jit() { diff --git a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml index fa175edcae6..5b79d6569bb 100644 --- a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml +++ b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "compiler_builtins", "gimli", @@ -140,9 +140,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "compiler_builtins", "memchr", diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index 5689bdee64d..2cc5d7777a6 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-08-08" +channel = "nightly-2023-09-06" components = ["rust-src", "rustc-dev", "llvm-tools"] diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index c163b854384..3fc462a39cc 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -45,6 +45,7 @@ rm tests/ui/proc-macro/quote-debug.rs rm tests/ui/proc-macro/no-missing-docs.rs rm tests/ui/rust-2018/proc-macro-crate-in-paths.rs rm tests/ui/proc-macro/allowed-signatures.rs +rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs # vendor intrinsics rm tests/ui/sse2.rs # cpuid not supported, so sse2 not detected @@ -114,6 +115,7 @@ rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants rm tests/ui/mir/mir_raw_fat_ptr.rs # same rm tests/ui/consts/issue-33537.rs # same rm tests/ui/layout/valid_range_oob.rs # different ICE message +rm tests/ui/const-generics/generic_const_exprs/issue-80742.rs # gives error instead of ICE with cg_clif rm tests/ui/consts/issue-miri-1910.rs # different error message rm tests/ui/consts/offset_ub.rs # same diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index 998263de584..b19b935a0fe 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -81,7 +81,7 @@ impl DebugContext { match tcx.sess.source_map().lookup_line(span.lo()) { Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.lines(|lines| lines[line]); + let line_pos = file.lines()[line]; let col = file.relative_position(span.lo()) - line_pos; (file, u64::try_from(line).unwrap() + 1, u64::from(col.to_u32()) + 1) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index fdd27a454e0..e62de6b6147 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -177,244 +177,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) }); } - "llvm.x86.sse2.psrli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrai.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrai.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.psrli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.psrai.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrli.q" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.q imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.q" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.q imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.pslli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.psrli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.psrai.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrai.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.pslli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } "llvm.x86.ssse3.pshuf.b.128" | "llvm.x86.avx2.pshuf.b" => { let (a, b) = match args { [a, b] => (a, b), @@ -506,14 +268,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted()); ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted()); } - "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => { - intrinsic_args!(fx, args => (mem_addr, a); intrinsic); - let mem_addr = mem_addr.load_scalar(fx); - - // FIXME correctly handle the unalignment - let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); - dest.write_cvalue(fx, a); - } "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { let a = match args { [a] => a, @@ -571,8 +325,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( // llvm.x86.avx2.vperm2i128 // llvm.x86.ssse3.pshuf.b.128 // llvm.x86.avx2.pshuf.b -// llvm.x86.avx2.psrli.w -// llvm.x86.sse2.psrli.w fn llvm_add_sub<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, diff --git a/compiler/rustc_codegen_gcc/src/debuginfo.rs b/compiler/rustc_codegen_gcc/src/debuginfo.rs index a81585d4128..d1bfd833cd8 100644 --- a/compiler/rustc_codegen_gcc/src/debuginfo.rs +++ b/compiler/rustc_codegen_gcc/src/debuginfo.rs @@ -55,7 +55,7 @@ impl<'gcc, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'gcc, 'tcx> { _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, _llfn: RValue<'gcc>, _mir: &mir::Body<'tcx>, - ) -> Option<FunctionDebugContext<Self::DIScope, Self::DILocation>> { + ) -> Option<FunctionDebugContext<'tcx, Self::DIScope, Self::DILocation>> { // TODO(antoyo) None } diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index aed4a8f3c85..ddaff36f24b 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -83,6 +83,8 @@ codegen_llvm_unknown_ctarget_feature_prefix = unknown feature specified for `-Ctarget-feature`: `{$feature}` .note = features must begin with a `+` to enable or `-` to disable it +codegen_llvm_unknown_debuginfo_compression = unknown debuginfo compression algorithm {$algorithm} - will fall back to uncompressed debuginfo + codegen_llvm_write_bytecode = failed to write bytecode to {$path}: {$err} codegen_llvm_write_ir = failed to write LLVM IR to {$path} diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 863cb7068f8..64587f98b8a 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -340,15 +340,50 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { }; for arg in args { + // Note that the exact number of arguments pushed here is carefully synchronized with + // code all over the place, both in the codegen_llvm and codegen_ssa crates. That's how + // other code then knows which LLVM argument(s) correspond to the n-th Rust argument. let llarg_ty = match &arg.mode { PassMode::Ignore => continue, - PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), + PassMode::Direct(_) => { + // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), + // and for Scalar ABIs the LLVM type is fully determined by `layout.abi`, + // guarnateeing that we generate ABI-compatible LLVM IR. Things get tricky for + // aggregates... + if matches!(arg.layout.abi, abi::Abi::Aggregate { .. }) { + // This really shouldn't happen, since `immediate_llvm_type` will use + // `layout.fields` to turn this Rust type into an LLVM type. This means all + // sorts of Rust type details leak into the ABI. However wasm sadly *does* + // currently use this mode so we have to allow it -- but we absolutely + // shouldn't let any more targets do that. + // (Also see <https://github.com/rust-lang/rust/issues/115666>.) + assert!( + matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64"), + "`PassMode::Direct` for aggregates only allowed on wasm targets\nProblematic type: {:#?}", + arg.layout, + ); + } + arg.layout.immediate_llvm_type(cx) + } PassMode::Pair(..) => { + // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), + // so for ScalarPair we can easily be sure that we are generating ABI-compatible + // LLVM IR. + assert!( + matches!(arg.layout.abi, abi::Abi::ScalarPair(..)), + "PassMode::Pair for type {}", + arg.layout.ty + ); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + assert!(arg.layout.is_unsized()); + // Construct the type of a (wide) pointer to `ty`, and pass its two fields. + // Any two ABI-compatible unsized types have the same metadata type and + // moreover the same metadata value leads to the same dynamic size and + // alignment, so this respects ABI compatibility. let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty); let ptr_layout = cx.layout_of(ptr_ty); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); @@ -360,6 +395,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { if *pad_i32 { llargument_tys.push(Reg::i32().llvm_type(cx)); } + // Compute the LLVM type we use for this function from the cast type. + // We assume here that ABI-compatible Rust types have the same cast type. cast.llvm_type(cx) } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => cx.type_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index c1d3392386c..5cf83b1accb 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,4 +1,6 @@ -use crate::back::write::{self, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers}; +use crate::back::write::{ + self, bitcode_section_name, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers, +}; use crate::errors::{ DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, }; @@ -120,6 +122,7 @@ fn prepare_lto( info!("adding bitcode from {}", name); match get_bitcode_slice_from_object_data( child.data(&*archive_data).expect("corrupt rlib"), + cgcx, ) { Ok(data) => { let module = SerializedModule::FromRlib(data.to_vec()); @@ -141,10 +144,29 @@ fn prepare_lto( Ok((symbols_below_threshold, upstream_modules)) } -fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], LtoBitcodeFromRlib> { +fn get_bitcode_slice_from_object_data<'a>( + obj: &'a [u8], + cgcx: &CodegenContext<LlvmCodegenBackend>, +) -> Result<&'a [u8], LtoBitcodeFromRlib> { + // We're about to assume the data here is an object file with sections, but if it's raw LLVM IR that + // won't work. Fortunately, if that's what we have we can just return the object directly, so we sniff + // the relevant magic strings here and return. + if obj.starts_with(b"\xDE\xC0\x17\x0B") || obj.starts_with(b"BC\xC0\xDE") { + return Ok(obj); + } + // We drop the "__LLVM," prefix here because on Apple platforms there's a notion of "segment name" + // which in the public API for sections gets treated as part of the section name, but internally + // in MachOObjectFile.cpp gets treated separately. + let section_name = bitcode_section_name(cgcx).trim_start_matches("__LLVM,"); let mut len = 0; - let data = - unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; + let data = unsafe { + llvm::LLVMRustGetSliceFromObjectDataByName( + obj.as_ptr(), + obj.len(), + section_name.as_ptr(), + &mut len, + ) + }; if !data.is_null() { assert!(len != 0); let bc = unsafe { slice::from_raw_parts(data, len) }; diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 47cc5bd52e2..1f394a7335c 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -5,13 +5,17 @@ use crate::back::profiling::{ use crate::base; use crate::common; use crate::errors::{ - CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode, + CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, + WithLlvmError, WriteBytecode, }; use crate::llvm::{self, DiagnosticInfo, PassManager}; use crate::llvm_util; use crate::type_::Type; use crate::LlvmCodegenBackend; use crate::ModuleLlvm; +use llvm::{ + LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, +}; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{ BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, @@ -216,6 +220,40 @@ pub fn target_machine_factory( let force_emulated_tls = sess.target.force_emulated_tls; + // copy the exe path, followed by path all into one buffer + // null terminating them so we can use them as null terminated strings + let args_cstr_buff = { + let mut args_cstr_buff: Vec<u8> = Vec::new(); + let exe_path = std::env::current_exe().unwrap_or_default(); + let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); + + args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); + args_cstr_buff.push(0); + + for arg in sess.expanded_args.iter() { + args_cstr_buff.extend_from_slice(arg.as_bytes()); + args_cstr_buff.push(0); + } + + args_cstr_buff + }; + + let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); + match sess.opts.debuginfo_compression { + rustc_session::config::DebugInfoCompression::Zlib => { + if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zlib" }); + } + } + rustc_session::config::DebugInfoCompression::Zstd => { + if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { + sess.emit_warning(UnknownCompression { algorithm: "zstd" }); + } + } + rustc_session::config::DebugInfoCompression::None => {} + }; + let debuginfo_compression = SmallCStr::new(&debuginfo_compression); + Arc::new(move |config: TargetMachineFactoryConfig| { let split_dwarf_file = path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; @@ -241,7 +279,10 @@ pub fn target_machine_factory( relax_elf_relocations, use_init_array, split_dwarf_file.as_ptr(), + debuginfo_compression.as_ptr(), force_emulated_tls, + args_cstr_buff.as_ptr() as *const c_char, + args_cstr_buff.len(), ) }; @@ -853,6 +894,27 @@ fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: asm } +fn target_is_apple(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { + cgcx.opts.target_triple.triple().contains("-ios") + || cgcx.opts.target_triple.triple().contains("-darwin") + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos") +} + +fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool { + cgcx.opts.target_triple.triple().contains("-aix") +} + +pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str { + if target_is_apple(cgcx) { + "__LLVM,__bitcode\0" + } else if target_is_aix(cgcx) { + ".ipa\0" + } else { + ".llvmbc\0" + } +} + /// Embed the bitcode of an LLVM module in the LLVM module itself. /// /// This is done primarily for iOS where it appears to be standard to compile C @@ -913,11 +975,8 @@ unsafe fn embed_bitcode( // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). - let is_aix = cgcx.opts.target_triple.triple().contains("-aix"); - let is_apple = cgcx.opts.target_triple.triple().contains("-ios") - || cgcx.opts.target_triple.triple().contains("-darwin") - || cgcx.opts.target_triple.triple().contains("-tvos") - || cgcx.opts.target_triple.triple().contains("-watchos"); + let is_aix = target_is_aix(cgcx); + let is_apple = target_is_apple(cgcx); if is_apple || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") @@ -932,13 +991,7 @@ unsafe fn embed_bitcode( ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { - "__LLVM,__bitcode\0" - } else if is_aix { - ".ipa\0" - } else { - ".llvmbc\0" - }; + let section = bitcode_section_name(cgcx); llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index ebf4ee4164f..5ca2942ac4e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -20,7 +20,7 @@ pub fn compute_mir_scopes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>, mir: &Body<'tcx>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, ) { // Find all scopes with variables defined in them. let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { @@ -51,7 +51,7 @@ fn make_mir_scope<'ll, 'tcx>( instance: Instance<'tcx>, mir: &Body<'tcx>, variables: &Option<BitSet<SourceScope>>, - debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, instantiated: &mut BitSet<SourceScope>, scope: SourceScope, ) { @@ -86,7 +86,7 @@ fn make_mir_scope<'ll, 'tcx>( let loc = cx.lookup_debug_loc(scope_data.span.lo()); let file_metadata = file_metadata(cx, &loc.file); - let dbg_scope = match scope_data.inlined { + let parent_dbg_scope = match scope_data.inlined { Some((callee, _)) => { // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. @@ -95,18 +95,22 @@ fn make_mir_scope<'ll, 'tcx>( ty::ParamEnv::reveal_all(), ty::EarlyBinder::bind(callee), ); - let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); - cx.dbg_scope_fn(callee, callee_fn_abi, None) + debug_context.inlined_function_scopes.entry(callee).or_insert_with(|| { + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) + }) } - None => unsafe { - llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_scope.dbg_scope, - file_metadata, - loc.line, - loc.col, - ) - }, + None => parent_scope.dbg_scope, + }; + + let dbg_scope = unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_dbg_scope, + file_metadata, + loc.line, + loc.col, + ) }; let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 2301f66dc28..52481a1090c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -263,7 +263,7 @@ impl CodegenCx<'_, '_> { pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { let (file, line, col) = match self.sess().source_map().lookup_line(pos) { Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.lines(|lines| lines[line]); + let line_pos = file.lines()[line]; // Use 1-based indexing. let line = (line + 1) as u32; @@ -292,7 +292,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: &'ll Value, mir: &mir::Body<'tcx>, - ) -> Option<FunctionDebugContext<&'ll DIScope, &'ll DILocation>> { + ) -> Option<FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>> { if self.sess().opts.debuginfo == DebugInfo::None { return None; } @@ -304,8 +304,10 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { file_start_pos: BytePos(0), file_end_pos: BytePos(0), }; - let mut fn_debug_context = - FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; + let mut fn_debug_context = FunctionDebugContext { + scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes), + inlined_function_scopes: Default::default(), + }; // Fill in all the scopes, with the information from the MIR body. compute_mir_scopes(self, instance, mir, &mut fn_debug_context); diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index fced6d504d2..264c273ba30 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -226,3 +226,9 @@ pub(crate) struct WriteBytecode<'a> { pub(crate) struct CopyBitcode { pub err: std::io::Error, } + +#[derive(Diagnostic)] +#[diag(codegen_llvm_unknown_debuginfo_compression)] +pub struct UnknownCompression { + pub algorithm: &'static str, +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 01cbf7d3b11..2ebfdae39e8 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2131,8 +2131,12 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, + DebugInfoCompression: *const c_char, ForceEmulatedTls: bool, + ArgsCstrBuff: *const c_char, + ArgsCstrBuffLen: usize, ) -> Option<&'static mut TargetMachine>; + pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, @@ -2319,6 +2323,12 @@ extern "C" { len: usize, out_len: &mut usize, ) -> *const u8; + pub fn LLVMRustGetSliceFromObjectDataByName( + data: *const u8, + len: usize, + name: *const u8, + out_len: &mut usize, + ) -> *const u8; pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub fn LLVMRustLinkerAdd( @@ -2357,6 +2367,10 @@ extern "C" { pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool; + pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; + + pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + pub fn LLVMRustGetSymbols( buf_ptr: *const u8, buf_len: usize, diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index f485af00bca..3bf98c46dea 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -343,6 +343,12 @@ pub struct CodegenContext<B: WriteBackendMethods> { pub split_debuginfo: rustc_target::spec::SplitDebuginfo, pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, + /// All commandline args used to invoke the compiler, with @file args fully expanded. + /// This will only be used within debug info, e.g. in the pdb file on windows + /// This is mainly useful for other tools that reads that debuginfo to figure out + /// how to call the compiler with the same arguments. + pub expanded_args: Vec<String>, + /// Handler to use for diagnostics produced during codegen. pub diag_emitter: SharedEmitter, /// LLVM optimizations for which we want to print remarks. @@ -1108,6 +1114,7 @@ fn start_executing_work<B: ExtraBackendMethods>( incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), coordinator_send, + expanded_args: tcx.sess.expanded_args.clone(), diag_emitter: shared_emitter.clone(), output_filenames: tcx.output_filenames(()).clone(), regular_module_config: regular_config, diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 526c16a59de..ac705a5f35c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -1,10 +1,12 @@ use crate::traits::*; +use rustc_data_structures::fx::FxHashMap; use rustc_index::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir; use rustc_middle::ty; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Instance; use rustc_middle::ty::Ty; use rustc_session::config::DebugInfo; use rustc_span::symbol::{kw, Symbol}; @@ -17,10 +19,13 @@ use super::{FunctionCx, LocalRef}; use std::ops::Range; -pub struct FunctionDebugContext<S, L> { +pub struct FunctionDebugContext<'tcx, S, L> { + /// Maps from source code to the corresponding debug info scope. pub scopes: IndexVec<mir::SourceScope, DebugScope<S, L>>, -} + /// Maps from an inlined function to its debug info declaration. + pub inlined_function_scopes: FxHashMap<Instance<'tcx>, S>, +} #[derive(Copy, Clone)] pub enum VariableKind { ArgumentVariable(usize /*index*/), @@ -484,54 +489,89 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { None }; - let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| { - let (var_ty, var_kind) = match var.value { + let var_ty = if let Some(ref fragment) = var.composite { + self.monomorphize(fragment.ty) + } else { + match var.value { mir::VarDebugInfoContents::Place(place) => { - let var_ty = self.monomorphized_place_ty(place.as_ref()); - let var_kind = if let Some(arg_index) = var.argument_index - && place.projection.is_empty() - { - let arg_index = arg_index as usize; - if target_is_msvc { - // ScalarPair parameters are spilled to the stack so they need to - // be marked as a `LocalVariable` for MSVC debuggers to visualize - // their data correctly. (See #81894 & #88625) - let var_ty_layout = self.cx.layout_of(var_ty); - if let Abi::ScalarPair(_, _) = var_ty_layout.abi { - VariableKind::LocalVariable - } else { - VariableKind::ArgumentVariable(arg_index) - } - } else { - // FIXME(eddyb) shouldn't `ArgumentVariable` indices be - // offset in closures to account for the hidden environment? - VariableKind::ArgumentVariable(arg_index) - } - } else { - VariableKind::LocalVariable - }; - (var_ty, var_kind) - } - mir::VarDebugInfoContents::Const(c) => { - let ty = self.monomorphize(c.ty()); - (ty, VariableKind::LocalVariable) + self.monomorphized_place_ty(place.as_ref()) } - mir::VarDebugInfoContents::Composite { ty, fragments: _ } => { - let ty = self.monomorphize(ty); - (ty, VariableKind::LocalVariable) + mir::VarDebugInfoContents::Const(c) => self.monomorphize(c.ty()), + } + }; + + let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| { + let var_kind = if let Some(arg_index) = var.argument_index + && var.composite.is_none() + && let mir::VarDebugInfoContents::Place(place) = var.value + && place.projection.is_empty() + { + let arg_index = arg_index as usize; + if target_is_msvc { + // ScalarPair parameters are spilled to the stack so they need to + // be marked as a `LocalVariable` for MSVC debuggers to visualize + // their data correctly. (See #81894 & #88625) + let var_ty_layout = self.cx.layout_of(var_ty); + if let Abi::ScalarPair(_, _) = var_ty_layout.abi { + VariableKind::LocalVariable + } else { + VariableKind::ArgumentVariable(arg_index) + } + } else { + // FIXME(eddyb) shouldn't `ArgumentVariable` indices be + // offset in closures to account for the hidden environment? + VariableKind::ArgumentVariable(arg_index) } + } else { + VariableKind::LocalVariable }; self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span) }); + let fragment = if let Some(ref fragment) = var.composite { + let var_layout = self.cx.layout_of(var_ty); + + let mut fragment_start = Size::ZERO; + let mut fragment_layout = var_layout; + + for elem in &fragment.projection { + match *elem { + mir::ProjectionElem::Field(field, _) => { + let i = field.index(); + fragment_start += fragment_layout.fields.offset(i); + fragment_layout = fragment_layout.field(self.cx, i); + } + _ => span_bug!( + var.source_info.span, + "unsupported fragment projection `{:?}`", + elem, + ), + } + } + + if fragment_layout.size == Size::ZERO { + // Fragment is a ZST, so does not represent anything. Avoid generating anything + // as this may conflict with a fragment that covers the entire variable. + continue; + } else if fragment_layout.size == var_layout.size { + // Fragment covers entire variable, so as far as + // DWARF is concerned, it's not really a fragment. + None + } else { + Some(fragment_start..fragment_start + fragment_layout.size) + } + } else { + None + }; + match var.value { mir::VarDebugInfoContents::Place(place) => { per_local[place.local].push(PerLocalVarDebugInfo { name: var.name, source_info: var.source_info, dbg_var, - fragment: None, + fragment, projection: place.projection, }); } @@ -547,51 +587,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, ); - bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, Size::ZERO, &[], None); - } - } - } - mir::VarDebugInfoContents::Composite { ty, ref fragments } => { - let var_ty = self.monomorphize(ty); - let var_layout = self.cx.layout_of(var_ty); - for fragment in fragments { - let mut fragment_start = Size::ZERO; - let mut fragment_layout = var_layout; - - for elem in &fragment.projection { - match *elem { - mir::ProjectionElem::Field(field, _) => { - let i = field.index(); - fragment_start += fragment_layout.fields.offset(i); - fragment_layout = fragment_layout.field(self.cx, i); - } - _ => span_bug!( - var.source_info.span, - "unsupported fragment projection `{:?}`", - elem, - ), - } + bx.dbg_var_addr( + dbg_var, + dbg_loc, + base.llval, + Size::ZERO, + &[], + fragment, + ); } - - let place = fragment.contents; - let fragment = if fragment_layout.size == Size::ZERO { - // Fragment is a ZST, so does not represent anything. - continue; - } else if fragment_layout.size == var_layout.size { - // Fragment covers entire variable, so as far as - // DWARF is concerned, it's not really a fragment. - None - } else { - Some(fragment_start..fragment_start + fragment_layout.size) - }; - - per_local[place.local].push(PerLocalVarDebugInfo { - name: var.name, - source_info: var.source_info, - dbg_var, - fragment, - projection: place.projection, - }); } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index bf937c2fc7c..c4408f2db4c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -46,7 +46,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { mir: &'tcx mir::Body<'tcx>, - debug_context: Option<FunctionDebugContext<Bx::DIScope, Bx::DILocation>>, + debug_context: Option<FunctionDebugContext<'tcx, Bx::DIScope, Bx::DILocation>>, llfn: Bx::Function, diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index 63fecaf34fd..4acc0ea076c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -26,7 +26,7 @@ pub trait DebugInfoMethods<'tcx>: BackendTypes { fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Function, mir: &mir::Body<'tcx>, - ) -> Option<FunctionDebugContext<Self::DIScope, Self::DILocation>>; + ) -> Option<FunctionDebugContext<'tcx, Self::DIScope, Self::DILocation>>; // FIXME(eddyb) find a common convention for all of the debuginfo-related // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index d8ad82d3da0..90f2b470179 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -796,6 +796,13 @@ where dest: &impl Writeable<'tcx, M::Provenance>, allow_transmute: bool, ) -> InterpResult<'tcx> { + // Generally for transmutation, data must be valid both at the old and new type. + // But if the types are the same, the 2nd validation below suffices. + if src.layout().ty != dest.layout().ty && M::enforce_validity(self, src.layout()) { + self.validate_operand(&src.to_op(self)?)?; + } + + // Do the actual copy. self.copy_op_no_validate(src, dest, allow_transmute)?; if M::enforce_validity(self, dest.layout()) { diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs index bc4edf1c4b6..eb4673c0edc 100644 --- a/compiler/rustc_const_eval/src/interpret/terminator.rs +++ b/compiler/rustc_const_eval/src/interpret/terminator.rs @@ -10,7 +10,7 @@ use rustc_middle::{ Instance, Ty, }, }; -use rustc_target::abi::call::{ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode}; +use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode}; use rustc_target::abi::{self, FieldIdx}; use rustc_target::spec::abi::Abi; @@ -291,32 +291,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { return true; } - match (caller_layout.abi, callee_layout.abi) { - // If both sides have Scalar/Vector/ScalarPair ABI, we can easily directly compare them. - // Different valid ranges are okay (the validity check will complain if this leads to - // invalid transmutes). Different signs are *not* okay on some targets (e.g. `extern - // "C"` on `s390x` where small integers are passed zero/sign-extended in large - // registers), so we generally reject them to increase portability. + match caller_layout.abi { + // For Scalar/Vector/ScalarPair ABI, we directly compare them. // NOTE: this is *not* a stable guarantee! It just reflects a property of our current // ABIs. It's also fragile; the same pair of types might be considered ABI-compatible // when used directly by-value but not considered compatible as a struct field or array // element. - (abi::Abi::Scalar(caller), abi::Abi::Scalar(callee)) => { - caller.primitive() == callee.primitive() + abi::Abi::Scalar(..) | abi::Abi::ScalarPair(..) | abi::Abi::Vector { .. } => { + caller_layout.abi.eq_up_to_validity(&callee_layout.abi) } - ( - abi::Abi::Vector { element: caller_element, count: caller_count }, - abi::Abi::Vector { element: callee_element, count: callee_count }, - ) => { - caller_element.primitive() == callee_element.primitive() - && caller_count == callee_count - } - (abi::Abi::ScalarPair(caller1, caller2), abi::Abi::ScalarPair(callee1, callee2)) => { - caller1.primitive() == callee1.primitive() - && caller2.primitive() == callee2.primitive() - } - (abi::Abi::Aggregate { .. }, abi::Abi::Aggregate { .. }) => { - // Aggregates are compatible only if they newtype-wrap the same type, or if they are both 1-ZST. + _ => { + // Everything else is compatible only if they newtype-wrap the same type, or if they are both 1-ZST. // (The latter part is needed to ensure e.g. that `struct Zst` is compatible with `struct Wrap((), Zst)`.) // This is conservative, but also means that our check isn't quite so heavily dependent on the `PassMode`, // which means having ABI-compatibility on one target is much more likely to imply compatibility for other targets. @@ -329,9 +314,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { == self.unfold_transparent(callee_layout).ty } } - // What remains is `Abi::Uninhabited` (which can never be passed anyway) and - // mismatching ABIs, that should all be rejected. - _ => false, } } @@ -340,40 +322,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { caller_abi: &ArgAbi<'tcx, Ty<'tcx>>, callee_abi: &ArgAbi<'tcx, Ty<'tcx>>, ) -> bool { - // When comparing the PassMode, we have to be smart about comparing the attributes. - let arg_attr_compat = |a1: &ArgAttributes, a2: &ArgAttributes| { - // There's only one regular attribute that matters for the call ABI: InReg. - // Everything else is things like noalias, dereferenceable, nonnull, ... - // (This also applies to pointee_size, pointee_align.) - if a1.regular.contains(ArgAttribute::InReg) != a2.regular.contains(ArgAttribute::InReg) - { - return false; - } - // We also compare the sign extension mode -- this could let the callee make assumptions - // about bits that conceptually were not even passed. - if a1.arg_ext != a2.arg_ext { - return false; - } - return true; - }; - let mode_compat = || match (&caller_abi.mode, &callee_abi.mode) { - (PassMode::Ignore, PassMode::Ignore) => true, // can still be reached for the return type - (PassMode::Direct(a1), PassMode::Direct(a2)) => arg_attr_compat(a1, a2), - (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => { - arg_attr_compat(a1, a2) && arg_attr_compat(b1, b2) - } - (PassMode::Cast(c1, pad1), PassMode::Cast(c2, pad2)) => c1 == c2 && pad1 == pad2, - ( - PassMode::Indirect { attrs: a1, extra_attrs: None, on_stack: s1 }, - PassMode::Indirect { attrs: a2, extra_attrs: None, on_stack: s2 }, - ) => arg_attr_compat(a1, a2) && s1 == s2, - ( - PassMode::Indirect { attrs: a1, extra_attrs: Some(e1), on_stack: s1 }, - PassMode::Indirect { attrs: a2, extra_attrs: Some(e2), on_stack: s2 }, - ) => arg_attr_compat(a1, a2) && arg_attr_compat(e1, e2) && s1 == s2, - _ => false, - }; - // Ideally `PassMode` would capture everything there is about argument passing, but that is // not the case: in `FnAbi::llvm_type`, also parts of the layout and type information are // used. So we need to check that *both* sufficiently agree to ensures the arguments are @@ -381,13 +329,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // For instance, `layout_compat` is needed to reject `i32` vs `f32`, which is not reflected // in `PassMode`. `mode_compat` is needed to reject `u8` vs `bool`, which have the same // `abi::Primitive` but different `arg_ext`. - if self.layout_compat(caller_abi.layout, callee_abi.layout) && mode_compat() { - // Something went very wrong if our checks don't even imply that the layout is the same. - assert!( - caller_abi.layout.size == callee_abi.layout.size - && caller_abi.layout.align.abi == callee_abi.layout.align.abi - && caller_abi.layout.is_sized() == callee_abi.layout.is_sized() - ); + if self.layout_compat(caller_abi.layout, callee_abi.layout) + && caller_abi.mode.eq_abi(&callee_abi.mode) + { + // Something went very wrong if our checks don't imply layout ABI compatibility. + assert!(caller_abi.layout.eq_abi(&callee_abi.layout)); return true; } else { trace!( diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 770c3f7f02c..2f5f2ad6534 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -6,13 +6,7 @@ use rustc_index::IndexVec; use rustc_infer::traits::Reveal; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::{ - traversal, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping, Local, Location, - MirPass, MirPhase, NonDivergingIntrinsic, NullOp, Operand, Place, PlaceElem, PlaceRef, - ProjectionElem, RetagKind, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, - Terminator, TerminatorKind, UnOp, UnwindAction, UnwindTerminateReason, VarDebugInfo, - VarDebugInfoContents, START_BLOCK, -}; +use rustc_middle::mir::*; use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeVisitableExt}; use rustc_mir_dataflow::impls::MaybeStorageLive; use rustc_mir_dataflow::storage::always_storage_live_locals; @@ -757,37 +751,37 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } fn visit_var_debug_info(&mut self, debuginfo: &VarDebugInfo<'tcx>) { - let check_place = |this: &mut Self, place: Place<'_>| { - if place.projection.iter().any(|p| !p.can_use_in_debuginfo()) { - this.fail( + if let Some(box VarDebugInfoFragment { ty, ref projection }) = debuginfo.composite { + if ty.is_union() || ty.is_enum() { + self.fail( START_BLOCK.start_location(), - format!("illegal place {:?} in debuginfo for {:?}", place, debuginfo.name), + format!("invalid type {ty:?} in debuginfo for {:?}", debuginfo.name), ); } - }; + if projection.is_empty() { + self.fail( + START_BLOCK.start_location(), + format!("invalid empty projection in debuginfo for {:?}", debuginfo.name), + ); + } + if projection.iter().any(|p| !matches!(p, PlaceElem::Field(..))) { + self.fail( + START_BLOCK.start_location(), + format!( + "illegal projection {:?} in debuginfo for {:?}", + projection, debuginfo.name + ), + ); + } + } match debuginfo.value { VarDebugInfoContents::Const(_) => {} VarDebugInfoContents::Place(place) => { - check_place(self, place); - } - VarDebugInfoContents::Composite { ty, ref fragments } => { - for f in fragments { - check_place(self, f.contents); - if ty.is_union() || ty.is_enum() { - self.fail( - START_BLOCK.start_location(), - format!("invalid type {ty:?} for composite debuginfo"), - ); - } - if f.projection.iter().any(|p| !matches!(p, PlaceElem::Field(..))) { - self.fail( - START_BLOCK.start_location(), - format!( - "illegal projection {:?} in debuginfo for {:?}", - f.projection, debuginfo.name - ), - ); - } + if place.projection.iter().any(|p| !p.can_use_in_debuginfo()) { + self.fail( + START_BLOCK.start_location(), + format!("illegal place {:?} in debuginfo for {:?}", place, debuginfo.name), + ); } } } diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index e82b0f6d496..8eda9043e35 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -61,6 +61,9 @@ pub use vec::{AppendOnlyIndexVec, AppendOnlyVec}; mod vec; +mod freeze; +pub use freeze::{FreezeLock, FreezeReadGuard, FreezeWriteGuard}; + mod mode { use super::Ordering; use std::sync::atomic::AtomicU8; diff --git a/compiler/rustc_data_structures/src/sync/freeze.rs b/compiler/rustc_data_structures/src/sync/freeze.rs new file mode 100644 index 00000000000..58ab91237f4 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/freeze.rs @@ -0,0 +1,171 @@ +use crate::sync::{AtomicBool, ReadGuard, RwLock, WriteGuard}; +#[cfg(parallel_compiler)] +use crate::sync::{DynSend, DynSync}; +use std::{ + cell::UnsafeCell, + intrinsics::likely, + marker::PhantomData, + ops::{Deref, DerefMut}, + sync::atomic::Ordering, +}; + +/// A type which allows mutation using a lock until +/// the value is frozen and can be accessed lock-free. +/// +/// Unlike `RwLock`, it can be used to prevent mutation past a point. +#[derive(Default)] +pub struct FreezeLock<T> { + data: UnsafeCell<T>, + frozen: AtomicBool, + + /// This lock protects writes to the `data` and `frozen` fields. + lock: RwLock<()>, +} + +#[cfg(parallel_compiler)] +unsafe impl<T: DynSync + DynSend> DynSync for FreezeLock<T> {} + +impl<T> FreezeLock<T> { + #[inline] + pub fn new(value: T) -> Self { + Self::with(value, false) + } + + #[inline] + pub fn frozen(value: T) -> Self { + Self::with(value, true) + } + + #[inline] + pub fn with(value: T, frozen: bool) -> Self { + Self { + data: UnsafeCell::new(value), + frozen: AtomicBool::new(frozen), + lock: RwLock::new(()), + } + } + + /// Clones the inner value along with the frozen state. + #[inline] + pub fn clone(&self) -> Self + where + T: Clone, + { + let lock = self.read(); + Self::with(lock.clone(), self.is_frozen()) + } + + #[inline] + pub fn is_frozen(&self) -> bool { + self.frozen.load(Ordering::Acquire) + } + + /// Get the inner value if frozen. + #[inline] + pub fn get(&self) -> Option<&T> { + if likely(self.frozen.load(Ordering::Acquire)) { + // SAFETY: This is frozen so the data cannot be modified. + unsafe { Some(&*self.data.get()) } + } else { + None + } + } + + #[inline] + pub fn read(&self) -> FreezeReadGuard<'_, T> { + FreezeReadGuard { + _lock_guard: if self.frozen.load(Ordering::Acquire) { + None + } else { + Some(self.lock.read()) + }, + lock: self, + } + } + + #[inline] + pub fn borrow(&self) -> FreezeReadGuard<'_, T> { + self.read() + } + + #[inline] + #[track_caller] + pub fn write(&self) -> FreezeWriteGuard<'_, T> { + self.try_write().expect("still mutable") + } + + #[inline] + pub fn try_write(&self) -> Option<FreezeWriteGuard<'_, T>> { + let _lock_guard = self.lock.write(); + // Use relaxed ordering since we're in the write lock. + if self.frozen.load(Ordering::Relaxed) { + None + } else { + Some(FreezeWriteGuard { _lock_guard, lock: self, marker: PhantomData }) + } + } + + #[inline] + pub fn freeze(&self) -> &T { + if !self.frozen.load(Ordering::Acquire) { + // Get the lock to ensure no concurrent writes and that we release the latest write. + let _lock = self.lock.write(); + self.frozen.store(true, Ordering::Release); + } + + // SAFETY: This is frozen so the data cannot be modified and shared access is sound. + unsafe { &*self.data.get() } + } +} + +/// A guard holding shared access to a `FreezeLock` which is in a locked state or frozen. +#[must_use = "if unused the FreezeLock may immediately unlock"] +pub struct FreezeReadGuard<'a, T> { + _lock_guard: Option<ReadGuard<'a, ()>>, + lock: &'a FreezeLock<T>, +} + +impl<'a, T: 'a> Deref for FreezeReadGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + // SAFETY: If `lock` is not frozen, `_lock_guard` holds the lock to the `UnsafeCell` so + // this has shared access until the `FreezeReadGuard` is dropped. If `lock` is frozen, + // the data cannot be modified and shared access is sound. + unsafe { &*self.lock.data.get() } + } +} + +/// A guard holding mutable access to a `FreezeLock` which is in a locked state or frozen. +#[must_use = "if unused the FreezeLock may immediately unlock"] +pub struct FreezeWriteGuard<'a, T> { + _lock_guard: WriteGuard<'a, ()>, + lock: &'a FreezeLock<T>, + marker: PhantomData<&'a mut T>, +} + +impl<'a, T> FreezeWriteGuard<'a, T> { + pub fn freeze(self) -> &'a T { + self.lock.frozen.store(true, Ordering::Release); + + // SAFETY: This is frozen so the data cannot be modified and shared access is sound. + unsafe { &*self.lock.data.get() } + } +} + +impl<'a, T: 'a> Deref for FreezeWriteGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + // SAFETY: `self._lock_guard` holds the lock to the `UnsafeCell` so this has shared access. + unsafe { &*self.lock.data.get() } + } +} + +impl<'a, T: 'a> DerefMut for FreezeWriteGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + // SAFETY: `self._lock_guard` holds the lock to the `UnsafeCell` so this has mutable access. + unsafe { &mut *self.lock.data.get() } + } +} diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 1a16759d7f9..e56347ab38e 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -313,6 +313,7 @@ fn run_compiler( override_queries: None, make_codegen_backend, registry: diagnostics_registry(), + expanded_args: args, }; match make_input(&early_error_handler, &matches.free) { diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs index d7a008f9a57..5d3b2f45166 100644 --- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs +++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs @@ -169,7 +169,7 @@ impl AnnotateSnippetEmitterWriter { .map(|line| { // Ensure the source file is present before we try // to load a string from it. - source_map.ensure_source_file_source_present(file.clone()); + source_map.ensure_source_file_source_present(&file); ( format!("{}", source_map.filename_for_diagnostics(&file.name)), source_string(file.clone(), &line), diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs index da108327ae7..58be74f887b 100644 --- a/compiler/rustc_errors/src/emitter.rs +++ b/compiler/rustc_errors/src/emitter.rs @@ -1193,7 +1193,7 @@ impl EmitterWriter { let will_be_emitted = |span: Span| { !span.is_dummy() && { let file = sm.lookup_source_file(span.hi()); - sm.ensure_source_file_source_present(file) + sm.ensure_source_file_source_present(&file) } }; @@ -1388,7 +1388,7 @@ impl EmitterWriter { // Print out the annotate source lines that correspond with the error for annotated_file in annotated_files { // we can't annotate anything if the source is unavailable. - if !sm.ensure_source_file_source_present(annotated_file.file.clone()) { + if !sm.ensure_source_file_source_present(&annotated_file.file) { if !self.short_message { // We'll just print an unannotated message. for (annotation_id, line) in annotated_file.lines.iter().enumerate() { diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs index 390bf28df09..38667c5ff81 100644 --- a/compiler/rustc_errors/src/json.rs +++ b/compiler/rustc_errors/src/json.rs @@ -558,7 +558,7 @@ impl DiagnosticSpanLine { .span_to_lines(span) .map(|lines| { // We can't get any lines if the source is unavailable. - if !je.sm.ensure_source_file_source_present(lines.file.clone()) { + if !je.sm.ensure_source_file_source_present(&lines.file) { return vec![]; } diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index 55c4ec66cd9..990bd2d1cc9 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -44,7 +44,7 @@ use rustc_fluent_macro::fluent_messages; pub use rustc_lint_defs::{pluralize, Applicability}; use rustc_span::source_map::SourceMap; pub use rustc_span::ErrorGuaranteed; -use rustc_span::{Loc, Span}; +use rustc_span::{Loc, Span, DUMMY_SP}; use std::borrow::Cow; use std::error::Report; @@ -273,7 +273,7 @@ impl CodeSuggestion { assert!(!lines.lines.is_empty() || bounding_span.is_dummy()); // We can't splice anything if the source is unavailable. - if !sm.ensure_source_file_source_present(lines.file.clone()) { + if !sm.ensure_source_file_source_present(&lines.file) { return None; } @@ -1754,7 +1754,7 @@ impl DelayedDiagnostic { BacktraceStatus::Captured => { let inner = &self.inner; self.inner.subdiagnostic(DelayedAtWithNewline { - span: inner.span.primary_span().unwrap(), + span: inner.span.primary_span().unwrap_or(DUMMY_SP), emitted_at: inner.emitted_at.clone(), note: self.note, }); @@ -1764,7 +1764,7 @@ impl DelayedDiagnostic { _ => { let inner = &self.inner; self.inner.subdiagnostic(DelayedAtWithoutNewline { - span: inner.span.primary_span().unwrap(), + span: inner.span.primary_span().unwrap_or(DUMMY_SP), emitted_at: inner.emitted_at.clone(), note: self.note, }); diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl index 597cae6ff33..50da2278312 100644 --- a/compiler/rustc_hir_analysis/messages.ftl +++ b/compiler/rustc_hir_analysis/messages.ftl @@ -222,6 +222,12 @@ hir_analysis_return_type_notation_on_non_rpitit = .note = function returns `{$ty}`, which is not compatible with associated type return bounds .label = this function must be `async` or return `impl Trait` +hir_analysis_rpitit_refined = impl trait in impl method signature does not match trait method signature + .suggestion = replace the return type so that it matches the trait + .label = return type from trait method defined here + .unmatched_bound_label = this bound is stronger than that defined on the trait + .note = add `#[allow(refining_impl_trait)]` if it is intended for this to be part of the public API of this crate + hir_analysis_self_in_impl_self = `Self` is not valid in the self type of an impl block .note = replace `Self` with a different type diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs index 6082d446979..ed4dde419c4 100644 --- a/compiler/rustc_hir_analysis/src/astconv/errors.rs +++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs @@ -110,16 +110,22 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { { // The fallback span is needed because `assoc_name` might be an `Fn()`'s `Output` without a // valid span, so we point at the whole path segment instead. - let span = if assoc_name.span != DUMMY_SP { assoc_name.span } else { span }; + let is_dummy = assoc_name.span == DUMMY_SP; + let mut err = struct_span_err!( self.tcx().sess, - span, + if is_dummy { span } else { assoc_name.span }, E0220, "associated type `{}` not found for `{}`", assoc_name, ty_param_name ); + if is_dummy { + err.span_label(span, format!("associated type `{assoc_name}` not found")); + return err.emit(); + } + let all_candidate_names: Vec<_> = all_candidates() .flat_map(|r| self.tcx().associated_items(r.def_id()).in_definition_order()) .filter_map(|item| { @@ -131,10 +137,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { }) .collect(); - if let (Some(suggested_name), true) = ( - find_best_match_for_name(&all_candidate_names, assoc_name.name, None), - assoc_name.span != DUMMY_SP, - ) { + if let Some(suggested_name) = + find_best_match_for_name(&all_candidate_names, assoc_name.name, None) + { err.span_suggestion( assoc_name.span, "there is an associated type with a similar name", @@ -172,10 +177,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { }) .collect(); - if let (Some(suggested_name), true) = ( - find_best_match_for_name(&wider_candidate_names, assoc_name.name, None), - assoc_name.span != DUMMY_SP, - ) { + if let Some(suggested_name) = + find_best_match_for_name(&wider_candidate_names, assoc_name.name, None) + { if let [best_trait] = visible_traits .iter() .filter(|trait_def_id| { @@ -197,7 +201,28 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { } } - err.span_label(span, format!("associated type `{assoc_name}` not found")); + // If we still couldn't find any associated type, and only one associated type exists, + // suggests using it. + + if all_candidate_names.len() == 1 { + // this should still compile, except on `#![feature(associated_type_defaults)]` + // where it could suggests `type A = Self::A`, thus recursing infinitely + let applicability = if self.tcx().features().associated_type_defaults { + Applicability::Unspecified + } else { + Applicability::MaybeIncorrect + }; + + err.span_suggestion( + assoc_name.span, + format!("`{ty_param_name}` has the following associated type"), + all_candidate_names.first().unwrap().to_string(), + applicability, + ); + } else { + err.span_label(assoc_name.span, format!("associated type `{assoc_name}` not found")); + } + err.emit() } diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs index bd0ab6463f0..92cc9759304 100644 --- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs +++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs @@ -28,6 +28,8 @@ use rustc_trait_selection::traits::{ use std::borrow::Cow; use std::iter; +mod refine; + /// Checks that a method from an impl conforms to the signature of /// the same method as declared in the trait. /// @@ -53,6 +55,12 @@ pub(super) fn compare_impl_method<'tcx>( impl_trait_ref, CheckImpliedWfMode::Check, )?; + refine::check_refining_return_position_impl_trait_in_trait( + tcx, + impl_m, + trait_m, + impl_trait_ref, + ); }; } diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs new file mode 100644 index 00000000000..a8149b634ef --- /dev/null +++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs @@ -0,0 +1,311 @@ +use rustc_data_structures::fx::FxIndexSet; +use rustc_hir as hir; +use rustc_hir::def_id::DefId; +use rustc_infer::infer::{outlives::env::OutlivesEnvironment, TyCtxtInferExt}; +use rustc_lint_defs::builtin::REFINING_IMPL_TRAIT; +use rustc_middle::traits::{ObligationCause, Reveal}; +use rustc_middle::ty::{ + self, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitor, +}; +use rustc_span::{Span, DUMMY_SP}; +use rustc_trait_selection::traits::{ + elaborate, normalize_param_env_or_error, outlives_bounds::InferCtxtExt, ObligationCtxt, +}; +use std::ops::ControlFlow; + +/// Check that an implementation does not refine an RPITIT from a trait method signature. +pub(super) fn check_refining_return_position_impl_trait_in_trait<'tcx>( + tcx: TyCtxt<'tcx>, + impl_m: ty::AssocItem, + trait_m: ty::AssocItem, + impl_trait_ref: ty::TraitRef<'tcx>, +) { + if !tcx.impl_method_has_trait_impl_trait_tys(impl_m.def_id) { + return; + } + // crate-private traits don't have any library guarantees, there's no need to do this check. + if !tcx.visibility(trait_m.container_id(tcx)).is_public() { + return; + } + + // If a type in the trait ref is private, then there's also no reason to to do this check. + let impl_def_id = impl_m.container_id(tcx); + for arg in impl_trait_ref.args { + if let Some(ty) = arg.as_type() + && let Some(self_visibility) = type_visibility(tcx, ty) + && !self_visibility.is_public() + { + return; + } + } + + let impl_m_args = ty::GenericArgs::identity_for_item(tcx, impl_m.def_id); + let trait_m_to_impl_m_args = impl_m_args.rebase_onto(tcx, impl_def_id, impl_trait_ref.args); + let bound_trait_m_sig = tcx.fn_sig(trait_m.def_id).instantiate(tcx, trait_m_to_impl_m_args); + let trait_m_sig = tcx.liberate_late_bound_regions(impl_m.def_id, bound_trait_m_sig); + // replace the self type of the trait ref with `Self` so that diagnostics render better. + let trait_m_sig_with_self_for_diag = tcx.liberate_late_bound_regions( + impl_m.def_id, + tcx.fn_sig(trait_m.def_id).instantiate( + tcx, + tcx.mk_args_from_iter( + [tcx.types.self_param.into()] + .into_iter() + .chain(trait_m_to_impl_m_args.iter().skip(1)), + ), + ), + ); + + let Ok(hidden_tys) = tcx.collect_return_position_impl_trait_in_trait_tys(impl_m.def_id) else { + // Error already emitted, no need to delay another. + return; + }; + + let mut collector = ImplTraitInTraitCollector { tcx, types: FxIndexSet::default() }; + trait_m_sig.visit_with(&mut collector); + + // Bound that we find on RPITITs in the trait signature. + let mut trait_bounds = vec![]; + // Bounds that we find on the RPITITs in the impl signature. + let mut impl_bounds = vec![]; + + for trait_projection in collector.types.into_iter().rev() { + let impl_opaque_args = trait_projection.args.rebase_onto(tcx, trait_m.def_id, impl_m_args); + let hidden_ty = hidden_tys[&trait_projection.def_id].instantiate(tcx, impl_opaque_args); + + // If the hidden type is not an opaque, then we have "refined" the trait signature. + let ty::Alias(ty::Opaque, impl_opaque) = *hidden_ty.kind() else { + report_mismatched_rpitit_signature( + tcx, + trait_m_sig_with_self_for_diag, + trait_m.def_id, + impl_m.def_id, + None, + ); + return; + }; + + // This opaque also needs to be from the impl method -- otherwise, + // it's a refinement to a TAIT. + if !tcx.hir().get_if_local(impl_opaque.def_id).map_or(false, |node| { + matches!( + node.expect_item().expect_opaque_ty().origin, + hir::OpaqueTyOrigin::AsyncFn(def_id) | hir::OpaqueTyOrigin::FnReturn(def_id) + if def_id == impl_m.def_id.expect_local() + ) + }) { + report_mismatched_rpitit_signature( + tcx, + trait_m_sig_with_self_for_diag, + trait_m.def_id, + impl_m.def_id, + None, + ); + return; + } + + trait_bounds.extend( + tcx.item_bounds(trait_projection.def_id).iter_instantiated(tcx, trait_projection.args), + ); + impl_bounds.extend(elaborate( + tcx, + tcx.explicit_item_bounds(impl_opaque.def_id) + .iter_instantiated_copied(tcx, impl_opaque.args), + )); + } + + let hybrid_preds = tcx + .predicates_of(impl_def_id) + .instantiate_identity(tcx) + .into_iter() + .chain(tcx.predicates_of(trait_m.def_id).instantiate_own(tcx, trait_m_to_impl_m_args)) + .map(|(clause, _)| clause); + let param_env = ty::ParamEnv::new(tcx.mk_clauses_from_iter(hybrid_preds), Reveal::UserFacing); + let param_env = normalize_param_env_or_error(tcx, param_env, ObligationCause::dummy()); + + let ref infcx = tcx.infer_ctxt().build(); + let ocx = ObligationCtxt::new(infcx); + + // Normalize the bounds. This has two purposes: + // + // 1. Project the RPITIT projections from the trait to the opaques on the impl, + // which means that they don't need to be mapped manually. + // + // 2. Project any other projections that show up in the bound. That makes sure that + // we don't consider `tests/ui/async-await/in-trait/async-associated-types.rs` + // to be refining. + let (trait_bounds, impl_bounds) = + ocx.normalize(&ObligationCause::dummy(), param_env, (trait_bounds, impl_bounds)); + + // Since we've normalized things, we need to resolve regions, since we'll + // possibly have introduced region vars during projection. We don't expect + // this resolution to have incurred any region errors -- but if we do, then + // just delay a bug. + let mut implied_wf_types = FxIndexSet::default(); + implied_wf_types.extend(trait_m_sig.inputs_and_output); + implied_wf_types.extend(ocx.normalize( + &ObligationCause::dummy(), + param_env, + trait_m_sig.inputs_and_output, + )); + if !ocx.select_all_or_error().is_empty() { + tcx.sess.delay_span_bug( + DUMMY_SP, + "encountered errors when checking RPITIT refinement (selection)", + ); + return; + } + let outlives_env = OutlivesEnvironment::with_bounds( + param_env, + infcx.implied_bounds_tys(param_env, impl_m.def_id.expect_local(), implied_wf_types), + ); + let errors = infcx.resolve_regions(&outlives_env); + if !errors.is_empty() { + tcx.sess.delay_span_bug( + DUMMY_SP, + "encountered errors when checking RPITIT refinement (regions)", + ); + return; + } + // Resolve any lifetime variables that may have been introduced during normalization. + let Ok((trait_bounds, impl_bounds)) = infcx.fully_resolve((trait_bounds, impl_bounds)) else { + tcx.sess.delay_span_bug( + DUMMY_SP, + "encountered errors when checking RPITIT refinement (resolution)", + ); + return; + }; + + // For quicker lookup, use an `IndexSet` + // (we don't use one earlier because it's not foldable..) + let trait_bounds = FxIndexSet::from_iter(trait_bounds); + + // Find any clauses that are present in the impl's RPITITs that are not + // present in the trait's RPITITs. This will trigger on trivial predicates, + // too, since we *do not* use the trait solver to prove that the RPITIT's + // bounds are not stronger -- we're doing a simple, syntactic compatibility + // check between bounds. This is strictly forwards compatible, though. + for (clause, span) in impl_bounds { + if !trait_bounds.contains(&clause) { + report_mismatched_rpitit_signature( + tcx, + trait_m_sig_with_self_for_diag, + trait_m.def_id, + impl_m.def_id, + Some(span), + ); + return; + } + } +} + +struct ImplTraitInTraitCollector<'tcx> { + tcx: TyCtxt<'tcx>, + types: FxIndexSet<ty::AliasTy<'tcx>>, +} + +impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitCollector<'tcx> { + type BreakTy = !; + + fn visit_ty(&mut self, ty: Ty<'tcx>) -> std::ops::ControlFlow<Self::BreakTy> { + if let ty::Alias(ty::Projection, proj) = *ty.kind() + && self.tcx.is_impl_trait_in_trait(proj.def_id) + { + if self.types.insert(proj) { + for (pred, _) in self + .tcx + .explicit_item_bounds(proj.def_id) + .iter_instantiated_copied(self.tcx, proj.args) + { + pred.visit_with(self)?; + } + } + ControlFlow::Continue(()) + } else { + ty.super_visit_with(self) + } + } +} + +fn report_mismatched_rpitit_signature<'tcx>( + tcx: TyCtxt<'tcx>, + trait_m_sig: ty::FnSig<'tcx>, + trait_m_def_id: DefId, + impl_m_def_id: DefId, + unmatched_bound: Option<Span>, +) { + let mapping = std::iter::zip( + tcx.fn_sig(trait_m_def_id).skip_binder().bound_vars(), + tcx.fn_sig(impl_m_def_id).skip_binder().bound_vars(), + ) + .filter_map(|(impl_bv, trait_bv)| { + if let ty::BoundVariableKind::Region(impl_bv) = impl_bv + && let ty::BoundVariableKind::Region(trait_bv) = trait_bv + { + Some((impl_bv, trait_bv)) + } else { + None + } + }) + .collect(); + + let mut return_ty = + trait_m_sig.output().fold_with(&mut super::RemapLateBound { tcx, mapping: &mapping }); + + if tcx.asyncness(impl_m_def_id).is_async() && tcx.asyncness(trait_m_def_id).is_async() { + let ty::Alias(ty::Projection, future_ty) = return_ty.kind() else { + bug!(); + }; + let Some(future_output_ty) = tcx + .explicit_item_bounds(future_ty.def_id) + .iter_instantiated_copied(tcx, future_ty.args) + .find_map(|(clause, _)| match clause.kind().no_bound_vars()? { + ty::ClauseKind::Projection(proj) => proj.term.ty(), + _ => None, + }) + else { + bug!() + }; + return_ty = future_output_ty; + } + + let (span, impl_return_span, pre, post) = + match tcx.hir().get_by_def_id(impl_m_def_id.expect_local()).fn_decl().unwrap().output { + hir::FnRetTy::DefaultReturn(span) => (tcx.def_span(impl_m_def_id), span, "-> ", " "), + hir::FnRetTy::Return(ty) => (ty.span, ty.span, "", ""), + }; + let trait_return_span = + tcx.hir().get_if_local(trait_m_def_id).map(|node| match node.fn_decl().unwrap().output { + hir::FnRetTy::DefaultReturn(_) => tcx.def_span(trait_m_def_id), + hir::FnRetTy::Return(ty) => ty.span, + }); + + let span = unmatched_bound.unwrap_or(span); + tcx.emit_spanned_lint( + REFINING_IMPL_TRAIT, + tcx.local_def_id_to_hir_id(impl_m_def_id.expect_local()), + span, + crate::errors::ReturnPositionImplTraitInTraitRefined { + impl_return_span, + trait_return_span, + pre, + post, + return_ty, + unmatched_bound, + }, + ); +} + +fn type_visibility<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<ty::Visibility<DefId>> { + match *ty.kind() { + ty::Ref(_, ty, _) => type_visibility(tcx, ty), + ty::Adt(def, args) => { + if def.is_fundamental() { + type_visibility(tcx, args.type_at(0)) + } else { + Some(tcx.visibility(def.did())) + } + } + _ => None, + } +} diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs index 9471ad9ca90..7614913f4bf 100644 --- a/compiler/rustc_hir_analysis/src/errors.rs +++ b/compiler/rustc_hir_analysis/src/errors.rs @@ -919,6 +919,22 @@ pub struct UnusedAssociatedTypeBounds { pub span: Span, } +#[derive(LintDiagnostic)] +#[diag(hir_analysis_rpitit_refined)] +#[note] +pub(crate) struct ReturnPositionImplTraitInTraitRefined<'tcx> { + #[suggestion(applicability = "maybe-incorrect", code = "{pre}{return_ty}{post}")] + pub impl_return_span: Span, + #[label] + pub trait_return_span: Option<Span>, + #[label(hir_analysis_unmatched_bound_label)] + pub unmatched_bound: Option<Span>, + + pub pre: &'static str, + pub post: &'static str, + pub return_ty: Ty<'tcx>, +} + #[derive(Diagnostic)] #[diag(hir_analysis_assoc_bound_on_const)] #[note] diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs index 4f95174f869..b0f333b79ca 100644 --- a/compiler/rustc_hir_analysis/src/lib.rs +++ b/compiler/rustc_hir_analysis/src/lib.rs @@ -237,6 +237,10 @@ pub fn check_crate(tcx: TyCtxt<'_>) -> Result<(), ErrorGuaranteed> { tcx.hir().for_each_module(|module| tcx.ensure().check_mod_item_types(module)) }); + // Freeze definitions as we don't add new ones at this point. This improves performance by + // allowing lock-free access to them. + tcx.untracked().definitions.freeze(); + // FIXME: Remove this when we implement creating `DefId`s // for anon constants during their parents' typeck. // Typeck all body owners in parallel will produce queries diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index aaabf1482e2..dabebe0adc0 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -764,13 +764,13 @@ impl<'tcx> InferCtxt<'tcx> { .collect(); vars.extend( (0..inner.int_unification_table().len()) - .map(|i| ty::IntVid { index: i as u32 }) + .map(|i| ty::IntVid::from_u32(i as u32)) .filter(|&vid| inner.int_unification_table().probe_value(vid).is_none()) .map(|v| Ty::new_int_var(self.tcx, v)), ); vars.extend( (0..inner.float_unification_table().len()) - .map(|i| ty::FloatVid { index: i as u32 }) + .map(|i| ty::FloatVid::from_u32(i as u32)) .filter(|&vid| inner.float_unification_table().probe_value(vid).is_none()) .map(|v| Ty::new_float_var(self.tcx, v)), ); diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 5b417e008cf..a0f0b530bae 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -279,6 +279,12 @@ pub struct Config { /// Registry of diagnostics codes. pub registry: Registry, + + /// All commandline args used to invoke the compiler, with @file args fully expanded. + /// This will only be used within debug info, e.g. in the pdb file on windows + /// This is mainly useful for other tools that reads that debuginfo to figure out + /// how to call the compiler with the same arguments. + pub expanded_args: Vec<String>, } // JUSTIFICATION: before session exists, only config @@ -317,6 +323,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se config.make_codegen_backend, registry.clone(), config.ice_file, + config.expanded_args, ); if let Some(parse_sess_created) = config.parse_sess_created { diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs index 2db7aa0e367..e0d9998d919 100644 --- a/compiler/rustc_interface/src/queries.rs +++ b/compiler/rustc_interface/src/queries.rs @@ -7,7 +7,9 @@ use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; -use rustc_data_structures::sync::{AppendOnlyIndexVec, Lrc, OnceLock, RwLock, WorkerLocal}; +use rustc_data_structures::sync::{ + AppendOnlyIndexVec, FreezeLock, Lrc, OnceLock, RwLock, WorkerLocal, +}; use rustc_hir::def_id::{StableCrateId, CRATE_DEF_ID, LOCAL_CRATE}; use rustc_hir::definitions::Definitions; use rustc_incremental::DepGraphFuture; @@ -197,7 +199,7 @@ impl<'tcx> Queries<'tcx> { self.codegen_backend().metadata_loader(), stable_crate_id, )) as _); - let definitions = RwLock::new(Definitions::new(stable_crate_id)); + let definitions = FreezeLock::new(Definitions::new(stable_crate_id)); let source_span = AppendOnlyIndexVec::new(); let _id = source_span.push(krate.spans.inner_span); debug_assert_eq!(_id, CRATE_DEF_ID); diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index b53ba251bcd..ccf8b5630d4 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -68,6 +68,7 @@ fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Se None, "", None, + Default::default(), ); (sess, cfg) } diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 5c9b39cdbe3..37e242c6e40 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -71,6 +71,7 @@ pub fn create_session( >, descriptions: Registry, ice_file: Option<PathBuf>, + expanded_args: Vec<String>, ) -> (Session, Box<dyn CodegenBackend>) { let codegen_backend = if let Some(make_codegen_backend) = make_codegen_backend { make_codegen_backend(&sopts) @@ -113,6 +114,7 @@ pub fn create_session( target_override, rustc_version_str().unwrap_or("unknown"), ice_file, + expanded_args, ); codegen_backend.init(&sess); diff --git a/compiler/rustc_lint/src/reference_casting.rs b/compiler/rustc_lint/src/reference_casting.rs index 883f6242b56..d540f473ae8 100644 --- a/compiler/rustc_lint/src/reference_casting.rs +++ b/compiler/rustc_lint/src/reference_casting.rs @@ -128,7 +128,11 @@ fn is_operation_we_care_about<'tcx>( fn is_cast_from_const_to_mut<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) -> bool { let e = e.peel_blocks(); - fn from_casts<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) -> Option<&'tcx Expr<'tcx>> { + fn from_casts<'tcx>( + cx: &LateContext<'tcx>, + e: &'tcx Expr<'tcx>, + need_check_freeze: &mut bool, + ) -> Option<&'tcx Expr<'tcx>> { // <expr> as *mut ... let mut e = if let ExprKind::Cast(e, t) = e.kind && let ty::RawPtr(TypeAndMut { mutbl: Mutability::Mut, .. }) = cx.typeck_results().node_type(t.hir_id).kind() { @@ -138,6 +142,14 @@ fn is_cast_from_const_to_mut<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) && let Some(def_id) = cx.typeck_results().type_dependent_def_id(e.hir_id) && cx.tcx.is_diagnostic_item(sym::ptr_cast_mut, def_id) { expr + // UnsafeCell::raw_get(<expr>) + } else if let ExprKind::Call(path, [arg]) = e.kind + && let ExprKind::Path(ref qpath) = path.kind + && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() + && cx.tcx.is_diagnostic_item(sym::unsafe_cell_raw_get, def_id) + { + *need_check_freeze = true; + arg } else { return None; }; @@ -160,11 +172,18 @@ fn is_cast_from_const_to_mut<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) { had_at_least_one_cast = true; expr - // ptr::from_ref(<expr>) + // ptr::from_ref(<expr>) or UnsafeCell::raw_get(<expr>) } else if let ExprKind::Call(path, [arg]) = e.kind && let ExprKind::Path(ref qpath) = path.kind && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() - && cx.tcx.is_diagnostic_item(sym::ptr_from_ref, def_id) { + && matches!( + cx.tcx.get_diagnostic_name(def_id), + Some(sym::ptr_from_ref | sym::unsafe_cell_raw_get) + ) + { + if cx.tcx.is_diagnostic_item(sym::unsafe_cell_raw_get, def_id) { + *need_check_freeze = true; + } return Some(arg); } else if had_at_least_one_cast { return Some(e); @@ -190,10 +209,25 @@ fn is_cast_from_const_to_mut<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) } } - let Some(e) = from_casts(cx, e).or_else(|| from_transmute(cx, e)) else { + let mut need_check_freeze = false; + let Some(e) = from_casts(cx, e, &mut need_check_freeze).or_else(|| from_transmute(cx, e)) + else { return false; }; let e = e.peel_blocks(); - matches!(cx.typeck_results().node_type(e.hir_id).kind(), ty::Ref(_, _, Mutability::Not)) + let node_type = cx.typeck_results().node_type(e.hir_id); + if let ty::Ref(_, inner_ty, Mutability::Not) = node_type.kind() { + // If an UnsafeCell method is involved we need to additionaly check the + // inner type for the presence of the Freeze trait (ie does NOT contain + // an UnsafeCell), since in that case we would incorrectly lint on valid casts. + // + // We also consider non concrete skeleton types (ie generics) + // to be an issue since there is no way to make it safe for abitrary types. + !need_check_freeze + || inner_ty.is_freeze(cx.tcx, cx.param_env) + || !inner_ty.has_concrete_skeleton() + } else { + false + } } diff --git a/compiler/rustc_lint/src/traits.rs b/compiler/rustc_lint/src/traits.rs index 56508a2a6cc..e812493b3dd 100644 --- a/compiler/rustc_lint/src/traits.rs +++ b/compiler/rustc_lint/src/traits.rs @@ -96,7 +96,7 @@ impl<'tcx> LateLintPass<'tcx> for DropTraitConstraints { }; let def_id = trait_predicate.trait_ref.def_id; if cx.tcx.lang_items().drop_trait() == Some(def_id) { - // Explicitly allow `impl Drop`, a drop-guards-as-Voldemort-type pattern. + // Explicitly allow `impl Drop`, a drop-guards-as-unnameable-type pattern. if trait_predicate.trait_ref.self_ty().is_impl_trait() { continue; } diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 7e745de4f1a..2567e273e11 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -3381,6 +3381,7 @@ declare_lint_pass! { PROC_MACRO_BACK_COMPAT, PROC_MACRO_DERIVE_RESOLUTION_FALLBACK, PUB_USE_OF_PRIVATE_EXTERN_CRATE, + REFINING_IMPL_TRAIT, RENAMED_AND_REMOVED_LINTS, REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, RUST_2021_INCOMPATIBLE_CLOSURE_CAPTURES, @@ -4363,7 +4364,7 @@ declare_lint! { /// pub struct S; /// } /// - /// pub fn get_voldemort() -> m::S { m::S } + /// pub fn get_unnameable() -> m::S { m::S } /// # fn main() {} /// ``` /// @@ -4448,7 +4449,6 @@ declare_lint! { /// ### Example /// /// ```rust,compile_fail - /// /// #![deny(ambiguous_glob_imports)] /// pub fn foo() -> u32 { /// use sub::*; @@ -4485,6 +4485,50 @@ declare_lint! { } declare_lint! { + /// The `refining_impl_trait` lint detects usages of return-position impl + /// traits in trait signatures which are refined by implementations. + /// + /// ### Example + /// + /// ```rust,compile_fail + /// #![feature(return_position_impl_trait_in_trait)] + /// #![deny(refining_impl_trait)] + /// + /// use std::fmt::Display; + /// + /// pub trait AsDisplay { + /// fn as_display(&self) -> impl Display; + /// } + /// + /// impl<'s> AsDisplay for &'s str { + /// fn as_display(&self) -> Self { + /// *self + /// } + /// } + /// + /// fn main() { + /// // users can observe that the return type of + /// // `<&str as AsDisplay>::as_display()` is `&str`. + /// let x: &str = "".as_display(); + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Return-position impl trait in traits (RPITITs) desugar to associated types, + /// and callers of methods for types where the implementation is known are + /// able to observe the types written in the impl signature. This may be + /// intended behavior, but may also pose a semver hazard for authors of libraries + /// who do not wish to make stronger guarantees about the types than what is + /// written in the trait signature. + pub REFINING_IMPL_TRAIT, + Warn, + "impl trait in impl method signature does not match trait method signature", +} + +declare_lint! { /// The `elided_lifetimes_in_associated_constant` lint detects elided lifetimes /// that were erroneously allowed in associated constants. /// diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 8f8f1d8c04f..fefb1e0fbe0 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -1,5 +1,6 @@ #include <stdio.h> +#include <cstddef> #include <iomanip> #include <vector> #include <set> @@ -9,6 +10,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CommandFlags.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/AssemblyAnnotationWriter.h" @@ -50,6 +52,8 @@ using namespace llvm; +static codegen::RegisterCodeGenFlags CGF; + typedef struct LLVMOpaquePass *LLVMPassRef; typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; @@ -406,7 +410,9 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( bool RelaxELFRelocations, bool UseInitArray, const char *SplitDwarfFile, - bool ForceEmulatedTls) { + const char *DebugInfoCompression, + bool ForceEmulatedTls, + const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { auto OptLevel = fromRust(RustOptLevel); auto RM = fromRust(RustReloc); @@ -421,7 +427,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( return nullptr; } - TargetOptions Options; + TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); Options.FloatABIType = FloatABI::Default; if (UseSoftFloat) { @@ -436,6 +442,16 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( if (SplitDwarfFile) { Options.MCOptions.SplitDwarfFile = SplitDwarfFile; } +#if LLVM_VERSION_GE(16, 0) + if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zlib; + } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) { + Options.CompressDebugSections = DebugCompressionType::Zstd; + } else if (!strcmp("none", DebugInfoCompression)) { + Options.CompressDebugSections = DebugCompressionType::None; + } +#endif + Options.RelaxELFRelocations = RelaxELFRelocations; Options.UseInitArray = UseInitArray; @@ -462,12 +478,48 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( Options.EmitStackSizeSection = EmitStackSizeSection; + + if (ArgsCstrBuff != nullptr) + { + int buffer_offset = 0; + assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); + + const size_t arg0_len = std::strlen(ArgsCstrBuff); + char* arg0 = new char[arg0_len + 1]; + memcpy(arg0, ArgsCstrBuff, arg0_len); + arg0[arg0_len] = '\0'; + buffer_offset += arg0_len + 1; + + const int num_cmd_arg_strings = + std::count(&ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); + + std::string* cmd_arg_strings = new std::string[num_cmd_arg_strings]; + for (int i = 0; i < num_cmd_arg_strings; ++i) + { + assert(buffer_offset < ArgsCstrBuffLen); + const int len = std::strlen(ArgsCstrBuff + buffer_offset); + cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); + buffer_offset += len + 1; + } + + assert(buffer_offset == ArgsCstrBuffLen); + + Options.MCOptions.Argv0 = arg0; + Options.MCOptions.CommandLineArgs = + llvm::ArrayRef<std::string>(cmd_arg_strings, num_cmd_arg_strings); + } + TargetMachine *TM = TheTarget->createTargetMachine( Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); return wrap(TM); } extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { + + MCTargetOptions& MCOptions = unwrap(TM)->Options.MCOptions; + delete[] MCOptions.Argv0; + delete[] MCOptions.CommandLineArgs.data(); + delete unwrap(TM); } @@ -1058,6 +1110,13 @@ extern "C" void LLVMRustPrintPasses() { extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, size_t Len) { auto PreserveFunctions = [=](const GlobalValue &GV) { + // Preserve LLVM-injected, ASAN-related symbols. + // See also https://github.com/rust-lang/rust/issues/113404. + if (GV.getName() == "___asan_globals_registered") { + return true; + } + + // Preserve symbols exported from Rust modules. for (size_t I = 0; I < Len; I++) { if (GV.getName() == Symbols[I]) { return true; @@ -1511,6 +1570,38 @@ LLVMRustGetBitcodeSliceFromObjectData(const char *data, return BitcodeOrError->getBufferStart(); } +// Find a section of an object file by name. Fail if the section is missing or +// empty. +extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, + size_t len, + const char *name, + size_t *out_len) { + *out_len = 0; + StringRef Data(data, len); + MemoryBufferRef Buffer(Data, ""); // The id is unused. + file_magic Type = identify_magic(Buffer.getBuffer()); + Expected<std::unique_ptr<object::ObjectFile>> ObjFileOrError = + object::ObjectFile::createObjectFile(Buffer, Type); + if (!ObjFileOrError) { + LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); + return nullptr; + } + for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { + Expected<StringRef> Name = Sec.getName(); + if (Name && *Name == name) { + Expected<StringRef> SectionOrError = Sec.getContents(); + if (!SectionOrError) { + LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); + return nullptr; + } + *out_len = SectionOrError->size(); + return SectionOrError->data(); + } + } + LLVMRustSetLastError("could not find requested section"); + return nullptr; +} + // Computes the LTO cache key for the provided 'ModId' in the given 'Data', // storing the result in 'KeyOut'. // Currently, this cache key is a SHA-1 hash of anything that could affect diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 70cdf3d6d23..4390486b0de 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -2044,3 +2044,19 @@ extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { } return false; } + +extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zlib::isAvailable(); +#else + return false; +#endif +} + +extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { +#if LLVM_VERSION_GE(16, 0) + return llvm::compression::zstd::isAvailable(); +#else + return false; +#endif +} diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 4f4351633a2..bace48cec44 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -280,8 +280,8 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for SpanData { // All of this logic ensures that the final result of deserialization is a 'normal' // Span that can be used without any additional trouble. let metadata_index = { - // Introduce a new scope so that we drop the 'lock()' temporary - match &*source_file.external_src.lock() { + // Introduce a new scope so that we drop the 'read()' temporary + match &*source_file.external_src.read() { ExternalSource::Foreign { metadata_index, .. } => *metadata_index, src => panic!("Unexpected external source {src:?}"), } diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index afcf08395bb..78a0f82db13 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -97,7 +97,7 @@ macro_rules! define_dep_nodes { // discriminants of the variants have been assigned consecutively from 0 // so that just the one comparison suffices to check that the u16 can be // transmuted to a DepKind. - const VARIANTS: u16 = { + pub const VARIANTS: u16 = { let deps: &[DepKind] = &[$(DepKind::$variant,)*]; let mut i = 0; while i < deps.len() { diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs index f79ce08b8ae..87436f9eeed 100644 --- a/compiler/rustc_middle/src/dep_graph/mod.rs +++ b/compiler/rustc_middle/src/dep_graph/mod.rs @@ -26,6 +26,7 @@ pub type DepKindStruct<'tcx> = rustc_query_system::dep_graph::DepKindStruct<TyCt impl rustc_query_system::dep_graph::DepKind for DepKind { const NULL: Self = DepKind::Null; const RED: Self = DepKind::Red; + const MAX: u16 = DepKind::VARIANTS - 1; fn debug_node(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}(", node.kind)?; @@ -68,6 +69,21 @@ impl rustc_query_system::dep_graph::DepKind for DepKind { op(icx.task_deps) }) } + + #[track_caller] + #[inline] + fn from_u16(u: u16) -> Self { + if u > Self::MAX { + panic!("Invalid DepKind {u}"); + } + // SAFETY: See comment on DepKind::VARIANTS + unsafe { std::mem::transmute(u) } + } + + #[inline] + fn to_u16(self) -> u16 { + self as u16 + } } impl<'tcx> DepContext for TyCtxt<'tcx> { diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs index 3e8f07ed233..e20a202561f 100644 --- a/compiler/rustc_middle/src/hir/map/mod.rs +++ b/compiler/rustc_middle/src/hir/map/mod.rs @@ -1207,7 +1207,7 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh { source_file_names.hash_stable(&mut hcx, &mut stable_hasher); debugger_visualizers.hash_stable(&mut hcx, &mut stable_hasher); if tcx.sess.opts.incremental_relative_spans() { - let definitions = tcx.definitions_untracked(); + let definitions = tcx.untracked().definitions.freeze(); let mut owner_spans: Vec<_> = krate .owners .iter_enumerated() diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 6484c30167c..c78b1e0e2f4 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1028,19 +1028,6 @@ pub enum VarDebugInfoContents<'tcx> { /// This `Place` only contains projection which satisfy `can_use_in_debuginfo`. Place(Place<'tcx>), Const(Constant<'tcx>), - /// The user variable's data is split across several fragments, - /// each described by a `VarDebugInfoFragment`. - /// See DWARF 5's "2.6.1.2 Composite Location Descriptions" - /// and LLVM's `DW_OP_LLVM_fragment` for more details on - /// the underlying debuginfo feature this relies on. - Composite { - /// Type of the original user variable. - /// This cannot contain a union or an enum. - ty: Ty<'tcx>, - /// All the parts of the original user variable, which ended - /// up in disjoint places, due to optimizations. - fragments: Vec<VarDebugInfoFragment<'tcx>>, - }, } impl<'tcx> Debug for VarDebugInfoContents<'tcx> { @@ -1048,19 +1035,16 @@ impl<'tcx> Debug for VarDebugInfoContents<'tcx> { match self { VarDebugInfoContents::Const(c) => write!(fmt, "{c}"), VarDebugInfoContents::Place(p) => write!(fmt, "{p:?}"), - VarDebugInfoContents::Composite { ty, fragments } => { - write!(fmt, "{ty:?}{{ ")?; - for f in fragments.iter() { - write!(fmt, "{f:?}, ")?; - } - write!(fmt, "}}") - } } } } -#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] +#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] pub struct VarDebugInfoFragment<'tcx> { + /// Type of the original user variable. + /// This cannot contain a union or an enum. + pub ty: Ty<'tcx>, + /// Where in the composite user variable this fragment is, /// represented as a "projection" into the composite variable. /// At lower levels, this corresponds to a byte/bit range. @@ -1071,29 +1055,10 @@ pub struct VarDebugInfoFragment<'tcx> { // to match on the discriminant, or by using custom type debuginfo // with non-overlapping variants for the composite variable. pub projection: Vec<PlaceElem<'tcx>>, - - /// Where the data for this fragment can be found. - /// This `Place` only contains projection which satisfy `can_use_in_debuginfo`. - pub contents: Place<'tcx>, -} - -impl Debug for VarDebugInfoFragment<'_> { - fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { - for elem in self.projection.iter() { - match elem { - ProjectionElem::Field(field, _) => { - write!(fmt, ".{:?}", field.index())?; - } - _ => bug!("unsupported fragment projection `{:?}`", elem), - } - } - - write!(fmt, " => {:?}", self.contents) - } } /// Debug information pertaining to a user variable. -#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] +#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] pub struct VarDebugInfo<'tcx> { pub name: Symbol, @@ -1102,6 +1067,13 @@ pub struct VarDebugInfo<'tcx> { /// (see `LocalDecl`'s `source_info` field for more details). pub source_info: SourceInfo, + /// The user variable's data is split across several fragments, + /// each described by a `VarDebugInfoFragment`. + /// See DWARF 5's "2.6.1.2 Composite Location Descriptions" + /// and LLVM's `DW_OP_LLVM_fragment` for more details on + /// the underlying debuginfo feature this relies on. + pub composite: Option<Box<VarDebugInfoFragment<'tcx>>>, + /// Where the data for this user variable is to be found. pub value: VarDebugInfoContents<'tcx>, @@ -1111,6 +1083,20 @@ pub struct VarDebugInfo<'tcx> { pub argument_index: Option<u16>, } +impl Debug for VarDebugInfo<'_> { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + if let Some(box VarDebugInfoFragment { ty, ref projection }) = self.composite { + pre_fmt_projection(&projection[..], fmt)?; + write!(fmt, "({}: {})", self.name, ty)?; + post_fmt_projection(&projection[..], fmt)?; + } else { + write!(fmt, "{}", self.name)?; + } + + write!(fmt, " => {:?}", self.value) + } +} + /////////////////////////////////////////////////////////////////////////// // BasicBlock @@ -1575,7 +1561,7 @@ impl<V, T> ProjectionElem<V, T> { /// need neither the `V` parameter for `Index` nor the `T` for `Field`. pub type ProjectionKind = ProjectionElem<(), ()>; -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct PlaceRef<'tcx> { pub local: Local, pub projection: &'tcx [PlaceElem<'tcx>], @@ -1751,69 +1737,90 @@ impl<'tcx> PlaceRef<'tcx> { } } +impl From<Local> for PlaceRef<'_> { + #[inline] + fn from(local: Local) -> Self { + PlaceRef { local, projection: &[] } + } +} + impl Debug for Place<'_> { fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { - for elem in self.projection.iter().rev() { - match elem { - ProjectionElem::OpaqueCast(_) - | ProjectionElem::Downcast(_, _) - | ProjectionElem::Field(_, _) => { - write!(fmt, "(").unwrap(); - } - ProjectionElem::Deref => { - write!(fmt, "(*").unwrap(); - } - ProjectionElem::Index(_) - | ProjectionElem::ConstantIndex { .. } - | ProjectionElem::Subslice { .. } => {} - } - } + self.as_ref().fmt(fmt) + } +} +impl Debug for PlaceRef<'_> { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + pre_fmt_projection(self.projection, fmt)?; write!(fmt, "{:?}", self.local)?; + post_fmt_projection(self.projection, fmt) + } +} - for elem in self.projection.iter() { - match elem { - ProjectionElem::OpaqueCast(ty) => { - write!(fmt, " as {ty})")?; - } - ProjectionElem::Downcast(Some(name), _index) => { - write!(fmt, " as {name})")?; - } - ProjectionElem::Downcast(None, index) => { - write!(fmt, " as variant#{index:?})")?; - } - ProjectionElem::Deref => { - write!(fmt, ")")?; - } - ProjectionElem::Field(field, ty) => { - write!(fmt, ".{:?}: {:?})", field.index(), ty)?; - } - ProjectionElem::Index(ref index) => { - write!(fmt, "[{index:?}]")?; - } - ProjectionElem::ConstantIndex { offset, min_length, from_end: false } => { - write!(fmt, "[{offset:?} of {min_length:?}]")?; - } - ProjectionElem::ConstantIndex { offset, min_length, from_end: true } => { - write!(fmt, "[-{offset:?} of {min_length:?}]")?; - } - ProjectionElem::Subslice { from, to, from_end: true } if to == 0 => { - write!(fmt, "[{from:?}:]")?; - } - ProjectionElem::Subslice { from, to, from_end: true } if from == 0 => { - write!(fmt, "[:-{to:?}]")?; - } - ProjectionElem::Subslice { from, to, from_end: true } => { - write!(fmt, "[{from:?}:-{to:?}]")?; - } - ProjectionElem::Subslice { from, to, from_end: false } => { - write!(fmt, "[{from:?}..{to:?}]")?; - } +fn pre_fmt_projection(projection: &[PlaceElem<'_>], fmt: &mut Formatter<'_>) -> fmt::Result { + for &elem in projection.iter().rev() { + match elem { + ProjectionElem::OpaqueCast(_) + | ProjectionElem::Downcast(_, _) + | ProjectionElem::Field(_, _) => { + write!(fmt, "(").unwrap(); } + ProjectionElem::Deref => { + write!(fmt, "(*").unwrap(); + } + ProjectionElem::Index(_) + | ProjectionElem::ConstantIndex { .. } + | ProjectionElem::Subslice { .. } => {} } + } - Ok(()) + Ok(()) +} + +fn post_fmt_projection(projection: &[PlaceElem<'_>], fmt: &mut Formatter<'_>) -> fmt::Result { + for &elem in projection.iter() { + match elem { + ProjectionElem::OpaqueCast(ty) => { + write!(fmt, " as {ty})")?; + } + ProjectionElem::Downcast(Some(name), _index) => { + write!(fmt, " as {name})")?; + } + ProjectionElem::Downcast(None, index) => { + write!(fmt, " as variant#{index:?})")?; + } + ProjectionElem::Deref => { + write!(fmt, ")")?; + } + ProjectionElem::Field(field, ty) => { + write!(fmt, ".{:?}: {:?})", field.index(), ty)?; + } + ProjectionElem::Index(ref index) => { + write!(fmt, "[{index:?}]")?; + } + ProjectionElem::ConstantIndex { offset, min_length, from_end: false } => { + write!(fmt, "[{offset:?} of {min_length:?}]")?; + } + ProjectionElem::ConstantIndex { offset, min_length, from_end: true } => { + write!(fmt, "[-{offset:?} of {min_length:?}]")?; + } + ProjectionElem::Subslice { from, to, from_end: true } if to == 0 => { + write!(fmt, "[{from:?}:]")?; + } + ProjectionElem::Subslice { from, to, from_end: true } if from == 0 => { + write!(fmt, "[:-{to:?}]")?; + } + ProjectionElem::Subslice { from, to, from_end: true } => { + write!(fmt, "[{from:?}:-{to:?}]")?; + } + ProjectionElem::Subslice { from, to, from_end: false } => { + write!(fmt, "[{from:?}..{to:?}]")?; + } + } } + + Ok(()) } /////////////////////////////////////////////////////////////////////////// @@ -2317,7 +2324,7 @@ impl<'tcx> ConstantKind<'tcx> { #[inline] pub fn try_to_scalar_int(self) -> Option<ScalarInt> { - Some(self.try_to_scalar()?.assert_int()) + self.try_to_scalar()?.try_to_int().ok() } #[inline] @@ -3056,6 +3063,6 @@ mod size_asserts { static_assert_size!(StatementKind<'_>, 16); static_assert_size!(Terminator<'_>, 104); static_assert_size!(TerminatorKind<'_>, 88); - static_assert_size!(VarDebugInfo<'_>, 80); + static_assert_size!(VarDebugInfo<'_>, 88); // tidy-alphabetical-end } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 773056e8a17..488526edb43 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -554,10 +554,7 @@ fn write_scope_tree( continue; } - let indented_debug_info = format!( - "{0:1$}debug {2} => {3:?};", - INDENT, indent, var_debug_info.name, var_debug_info.value, - ); + let indented_debug_info = format!("{0:1$}debug {2:?};", INDENT, indent, var_debug_info); if tcx.sess.opts.unstable_opts.mir_include_spans { writeln!( diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 87b04aabe6a..61244b94289 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -838,12 +838,20 @@ macro_rules! make_mir_visitor { let VarDebugInfo { name: _, source_info, + composite, value, argument_index: _, } = var_debug_info; self.visit_source_info(source_info); let location = Location::START; + if let Some(box VarDebugInfoFragment { ref $($mutability)? ty, ref $($mutability)? projection }) = composite { + self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); + for elem in projection { + let ProjectionElem::Field(_, ty) = elem else { bug!() }; + self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); + } + } match value { VarDebugInfoContents::Const(c) => self.visit_constant(c, location), VarDebugInfoContents::Place(place) => @@ -852,17 +860,6 @@ macro_rules! make_mir_visitor { PlaceContext::NonUse(NonUseContext::VarDebugInfo), location ), - VarDebugInfoContents::Composite { ty, fragments } => { - // FIXME(eddyb) use a better `TyContext` here. - self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); - for VarDebugInfoFragment { projection: _, contents } in fragments { - self.visit_place( - contents, - PlaceContext::NonUse(NonUseContext::VarDebugInfo), - location, - ); - } - } } } diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index 64853bd9612..154c930dc0f 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -692,7 +692,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span { let len = BytePos::decode(decoder); let file_lo = decoder.file_index_to_file(file_lo_index); - let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo); + let lo = file_lo.lines()[line_lo - 1] + col_lo; let lo = file_lo.absolute_position(lo); let hi = lo + len; diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs index b16163edf14..9d99344d5bd 100644 --- a/compiler/rustc_middle/src/ty/consts/int.rs +++ b/compiler/rustc_middle/src/ty/consts/int.rs @@ -227,6 +227,11 @@ impl ScalarInt { } #[inline] + pub fn try_from_target_usize(i: impl Into<u128>, tcx: TyCtxt<'_>) -> Option<Self> { + Self::try_from_uint(i, tcx.data_layout.pointer_size) + } + + #[inline] pub fn assert_bits(self, target_size: Size) -> u128 { self.to_bits(target_size).unwrap_or_else(|size| { bug!("expected int of size {}, but got size {}", target_size.bytes(), size.bytes()) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index e2f9e299566..90d847804f0 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -39,7 +39,9 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::sync::{self, Lock, Lrc, MappedReadGuard, ReadGuard, WorkerLocal}; +use rustc_data_structures::sync::{ + self, FreezeReadGuard, Lock, Lrc, MappedReadGuard, ReadGuard, WorkerLocal, +}; use rustc_data_structures::unord::UnordSet; use rustc_errors::{ DecorateLint, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, MultiSpan, @@ -964,8 +966,8 @@ impl<'tcx> TyCtxt<'tcx> { i += 1; } - // Leak a read lock once we finish iterating on definitions, to prevent adding new ones. - definitions.leak(); + // Freeze definitions once we finish iterating on them, to prevent adding new ones. + definitions.freeze(); }) } @@ -974,10 +976,9 @@ impl<'tcx> TyCtxt<'tcx> { // definitions change. self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE); - // Leak a read lock once we start iterating on definitions, to prevent adding new ones + // Freeze definitions once we start iterating on them, to prevent adding new ones // while iterating. If some query needs to add definitions, it should be `ensure`d above. - let definitions = self.untracked.definitions.leak(); - definitions.def_path_table() + self.untracked.definitions.freeze().def_path_table() } pub fn def_path_hash_to_def_index_map( @@ -986,10 +987,9 @@ impl<'tcx> TyCtxt<'tcx> { // Create a dependency to the crate to be sure we re-execute this when the amount of // definitions change. self.ensure().hir_crate(()); - // Leak a read lock once we start iterating on definitions, to prevent adding new ones + // Freeze definitions once we start iterating on them, to prevent adding new ones // while iterating. If some query needs to add definitions, it should be `ensure`d above. - let definitions = self.untracked.definitions.leak(); - definitions.def_path_hash_to_def_index_map() + self.untracked.definitions.freeze().def_path_hash_to_def_index_map() } /// Note that this is *untracked* and should only be used within the query @@ -1006,7 +1006,7 @@ impl<'tcx> TyCtxt<'tcx> { /// Note that this is *untracked* and should only be used within the query /// system if the result is otherwise tracked through queries #[inline] - pub fn definitions_untracked(self) -> ReadGuard<'tcx, Definitions> { + pub fn definitions_untracked(self) -> FreezeReadGuard<'tcx, Definitions> { self.untracked.definitions.read() } diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs index 97dab5cb47e..53a3d5fc6be 100644 --- a/compiler/rustc_middle/src/ty/generic_args.rs +++ b/compiler/rustc_middle/src/ty/generic_args.rs @@ -440,7 +440,7 @@ impl<'tcx> GenericArgs<'tcx> { target_args: GenericArgsRef<'tcx>, ) -> GenericArgsRef<'tcx> { let defs = tcx.generics_of(source_ancestor); - tcx.mk_args_from_iter(target_args.iter().chain(self.iter().skip(defs.params.len()))) + tcx.mk_args_from_iter(target_args.iter().chain(self.iter().skip(defs.count()))) } pub fn truncate_to(&self, tcx: TyCtxt<'tcx>, generics: &ty::Generics) -> GenericArgsRef<'tcx> { diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index ac0c88468fa..12254ed1a4d 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -1123,6 +1123,17 @@ pub trait PrettyPrinter<'tcx>: } } + if self.tcx().features().return_type_notation + && let Some(ty::ImplTraitInTraitData::Trait { fn_def_id, .. }) = self.tcx().opt_rpitit_info(def_id) + && let ty::Alias(_, alias_ty) = self.tcx().fn_sig(fn_def_id).skip_binder().output().skip_binder().kind() + && alias_ty.def_id == def_id + { + let num_args = self.tcx().generics_of(fn_def_id).count(); + write!(self, " {{ ")?; + self = self.print_def_path(fn_def_id, &args[..num_args])?; + write!(self, "() }}")?; + } + Ok(self) } @@ -1239,21 +1250,18 @@ pub trait PrettyPrinter<'tcx>: .generics_of(principal.def_id) .own_args_no_defaults(cx.tcx(), principal.args); - let mut projections = predicates.projection_bounds(); - - let mut args = args.iter().cloned(); - let arg0 = args.next(); - let projection0 = projections.next(); - if arg0.is_some() || projection0.is_some() { - let args = arg0.into_iter().chain(args); - let projections = projection0.into_iter().chain(projections); + let mut projections: Vec<_> = predicates.projection_bounds().collect(); + projections.sort_by_cached_key(|proj| { + cx.tcx().item_name(proj.item_def_id()).to_string() + }); + if !args.is_empty() || !projections.is_empty() { p!(generic_delimiters(|mut cx| { - cx = cx.comma_sep(args)?; - if arg0.is_some() && projection0.is_some() { + cx = cx.comma_sep(args.iter().copied())?; + if !args.is_empty() && !projections.is_empty() { write!(cx, ", ")?; } - cx.comma_sep(projections) + cx.comma_sep(projections.iter().copied()) })); } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 0291cdd6c57..fbbdfe5f14f 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2857,7 +2857,7 @@ impl<'tcx> Ty<'tcx> { | ty::Uint(..) | ty::Float(..) => true, - // The voldemort ZSTs are fine. + // ZST which can't be named are fine. ty::FnDef(..) => true, ty::Array(element_ty, _len) => element_ty.is_trivially_pure_clone_copy(), diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs index f83c62fd580..e2ab2cb90c7 100644 --- a/compiler/rustc_mir_build/src/build/custom/parse.rs +++ b/compiler/rustc_mir_build/src/build/custom/parse.rs @@ -241,6 +241,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> { let dbginfo = VarDebugInfo { name, source_info: SourceInfo { span, scope: self.source_scope }, + composite: None, argument_index: None, value, }; diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs index 5ec216cea61..7e81c8b50c2 100644 --- a/compiler/rustc_mir_build/src/build/matches/mod.rs +++ b/compiler/rustc_mir_build/src/build/matches/mod.rs @@ -2287,6 +2287,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { name, source_info: debug_source_info, value: VarDebugInfoContents::Place(for_arm_body.into()), + composite: None, argument_index: None, }); let locals = if has_guard.0 { @@ -2306,6 +2307,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { name, source_info: debug_source_info, value: VarDebugInfoContents::Place(ref_for_guard.into()), + composite: None, argument_index: None, }); LocalsForNode::ForGuard { ref_for_guard, for_arm_body } diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index e614046e83e..4e10916ad61 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -823,6 +823,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { name, source_info: SourceInfo::outermost(captured_place.var_ident.span), value: VarDebugInfoContents::Place(use_place), + composite: None, argument_index: None, }); @@ -852,6 +853,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { name, source_info, value: VarDebugInfoContents::Place(arg_local.into()), + composite: None, argument_index: Some(argument_index as u16 + 1), }); } diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 6872ef5e985..299bf692307 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -581,6 +581,14 @@ impl<V: Clone + HasTop + HasBottom> State<V> { } } + /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V { + match map.find_len(place) { + Some(place) => self.get_idx(place, map), + None => V::TOP, + } + } + /// Retrieve the value stored for a place index, or ⊤ if it is not tracked. pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V { match &self.0 { @@ -626,45 +634,36 @@ pub struct Map { } impl Map { - fn new() -> Self { - Self { + /// Returns a map that only tracks places whose type has scalar layout. + /// + /// This is currently the only way to create a [`Map`]. The way in which the tracked places are + /// chosen is an implementation detail and may not be relied upon (other than that their type + /// are scalars). + pub fn new<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, value_limit: Option<usize>) -> Self { + let mut map = Self { locals: IndexVec::new(), projections: FxHashMap::default(), places: IndexVec::new(), value_count: 0, inner_values: IndexVec::new(), inner_values_buffer: Vec::new(), - } - } - - /// Returns a map that only tracks places whose type passes the filter. - /// - /// This is currently the only way to create a [`Map`]. The way in which the tracked places are - /// chosen is an implementation detail and may not be relied upon (other than that their type - /// passes the filter). - pub fn from_filter<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - filter: impl Fn(Ty<'tcx>) -> bool, - value_limit: Option<usize>, - ) -> Self { - let mut map = Self::new(); + }; let exclude = excluded_locals(body); - map.register_with_filter(tcx, body, filter, exclude, value_limit); + map.register(tcx, body, exclude, value_limit); debug!("registered {} places ({} nodes in total)", map.value_count, map.places.len()); map } - /// Register all non-excluded places that pass the filter. - fn register_with_filter<'tcx>( + /// Register all non-excluded places that have scalar layout. + fn register<'tcx>( &mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - filter: impl Fn(Ty<'tcx>) -> bool, exclude: BitSet<Local>, value_limit: Option<usize>, ) { let mut worklist = VecDeque::with_capacity(value_limit.unwrap_or(body.local_decls.len())); + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); // Start by constructing the places for each bare local. self.locals = IndexVec::from_elem(None, &body.local_decls); @@ -679,7 +678,7 @@ impl Map { self.locals[local] = Some(place); // And push the eventual children places to the worklist. - self.register_children(tcx, place, decl.ty, &filter, &mut worklist); + self.register_children(tcx, param_env, place, decl.ty, &mut worklist); } // `place.elem1.elem2` with type `ty`. @@ -702,7 +701,7 @@ impl Map { } // And push the eventual children places to the worklist. - self.register_children(tcx, place, ty, &filter, &mut worklist); + self.register_children(tcx, param_env, place, ty, &mut worklist); } // Pre-compute the tree of ValueIndex nested in each PlaceIndex. @@ -732,42 +731,52 @@ impl Map { fn register_children<'tcx>( &mut self, tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, place: PlaceIndex, ty: Ty<'tcx>, - filter: &impl Fn(Ty<'tcx>) -> bool, worklist: &mut VecDeque<(PlaceIndex, Option<TrackElem>, TrackElem, Ty<'tcx>)>, ) { // Allocate a value slot if it doesn't have one, and the user requested one. - if self.places[place].value_index.is_none() && filter(ty) { + assert!(self.places[place].value_index.is_none()); + if tcx.layout_of(param_env.and(ty)).map_or(false, |layout| layout.abi.is_scalar()) { self.places[place].value_index = Some(self.value_count.into()); self.value_count += 1; } // For enums, directly create the `Discriminant`, as that's their main use. if ty.is_enum() { - let discr_ty = ty.discriminant_ty(tcx); - if filter(discr_ty) { - let discr = *self - .projections - .entry((place, TrackElem::Discriminant)) - .or_insert_with(|| { - // Prepend new child to the linked list. - let next = self.places.push(PlaceInfo::new(Some(TrackElem::Discriminant))); - self.places[next].next_sibling = self.places[place].first_child; - self.places[place].first_child = Some(next); - next - }); - - // Allocate a value slot if it doesn't have one. - if self.places[discr].value_index.is_none() { - self.places[discr].value_index = Some(self.value_count.into()); - self.value_count += 1; - } - } + // Prepend new child to the linked list. + let discr = self.places.push(PlaceInfo::new(Some(TrackElem::Discriminant))); + self.places[discr].next_sibling = self.places[place].first_child; + self.places[place].first_child = Some(discr); + let old = self.projections.insert((place, TrackElem::Discriminant), discr); + assert!(old.is_none()); + + // Allocate a value slot since it doesn't have one. + assert!(self.places[discr].value_index.is_none()); + self.places[discr].value_index = Some(self.value_count.into()); + self.value_count += 1; + } + + if let Some(ref_ty) = ty.builtin_deref(true) && let ty::Slice(..) = ref_ty.ty.kind() { + assert!(self.places[place].value_index.is_none(), "slices are not scalars"); + + // Prepend new child to the linked list. + let len = self.places.push(PlaceInfo::new(Some(TrackElem::DerefLen))); + self.places[len].next_sibling = self.places[place].first_child; + self.places[place].first_child = Some(len); + + let old = self.projections.insert((place, TrackElem::DerefLen), len); + assert!(old.is_none()); + + // Allocate a value slot since it doesn't have one. + assert!( self.places[len].value_index.is_none() ); + self.places[len].value_index = Some(self.value_count.into()); + self.value_count += 1; } // Recurse with all fields of this place. - iter_fields(ty, tcx, ty::ParamEnv::reveal_all(), |variant, field, ty| { + iter_fields(ty, tcx, param_env, |variant, field, ty| { worklist.push_back(( place, variant.map(TrackElem::Variant), @@ -834,6 +843,11 @@ impl Map { self.find_extra(place, [TrackElem::Discriminant]) } + /// Locates the given place and applies `DerefLen`, if it exists in the tree. + pub fn find_len(&self, place: PlaceRef<'_>) -> Option<PlaceIndex> { + self.find_extra(place, [TrackElem::DerefLen]) + } + /// Iterate over all direct children. pub fn children(&self, parent: PlaceIndex) -> impl Iterator<Item = PlaceIndex> + '_ { Children::new(self, parent) @@ -985,6 +999,8 @@ pub enum TrackElem { Field(FieldIdx), Variant(VariantIdx), Discriminant, + // Length of a slice. + DerefLen, } impl<V, T> TryFrom<ProjectionElem<V, T>> for TrackElem { @@ -1124,6 +1140,9 @@ fn debug_with_context_rec<V: Debug + Eq>( format!("{}.{}", place_str, field.index()) } } + TrackElem::DerefLen => { + format!("Len(*{})", place_str) + } }; debug_with_context_rec(child, &child_place_str, new, old, map, f)?; } diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl index 2598eb2ed09..5a99afc45b0 100644 --- a/compiler/rustc_mir_transform/messages.ftl +++ b/compiler/rustc_mir_transform/messages.ftl @@ -42,8 +42,6 @@ mir_transform_requires_unsafe = {$details} is unsafe and requires unsafe {$op_in } .not_inherited = items do not inherit unsafety from separate enclosing items -mir_transform_simd_shuffle_last_const = last argument of `simd_shuffle` is required to be a `const` item - mir_transform_target_feature_call_label = call to function with `#[target_feature]` mir_transform_target_feature_call_note = can only be called if the required target features are available diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 0fce9cb19a8..d5af321d726 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -483,7 +483,7 @@ fn unsafety_check_result(tcx: TyCtxt<'_>, def: LocalDefId) -> &UnsafetyCheckResu // `mir_built` force this. let body = &tcx.mir_built(def).borrow(); - if body.is_custom_mir() { + if body.is_custom_mir() || body.tainted_by_errors.is_some() { return tcx.arena.alloc(UnsafetyCheckResult { violations: Vec::new(), used_unsafe_blocks: Default::default(), diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 4b51beed095..b52827a1e88 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -39,6 +39,10 @@ pub struct ConstProp; impl<'tcx> MirLint<'tcx> for ConstProp { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { + if body.tainted_by_errors.is_some() { + return; + } + // will be evaluated by miri and produce its errors there if body.source.promoted.is_some() { return; diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index aa205655f9d..56365c5d474 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -1,5 +1,6 @@ use super::*; +use rustc_data_structures::captures::Captures; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{self, Body, Coverage, CoverageInfo}; use rustc_middle::query::Providers; @@ -12,15 +13,10 @@ pub(crate) fn provide(providers: &mut Providers) { providers.covered_code_regions = |tcx, def_id| covered_code_regions(tcx, def_id); } -/// The `num_counters` argument to `llvm.instrprof.increment` is the max counter_id + 1, or in -/// other words, the number of counter value references injected into the MIR (plus 1 for the -/// reserved `ZERO` counter, which uses counter ID `0` when included in an expression). Injected -/// counters have a counter ID from `1..num_counters-1`. -/// -/// `num_expressions` is the number of counter expressions added to the MIR body. -/// -/// Both `num_counters` and `num_expressions` are used to initialize new vectors, during backend -/// code generate, to lookup counters and expressions by simple u32 indexes. +/// Coverage codegen needs to know the total number of counter IDs and expression IDs that have +/// been used by a function's coverage mappings. These totals are used to create vectors to hold +/// the relevant counter and expression data, and the maximum counter ID (+ 1) is also needed by +/// the `llvm.instrprof.increment` intrinsic. /// /// MIR optimization may split and duplicate some BasicBlock sequences, or optimize out some code /// including injected counters. (It is OK if some counters are optimized out, but those counters @@ -28,71 +24,51 @@ pub(crate) fn provide(providers: &mut Providers) { /// calls may not work; but computing the number of counters or expressions by adding `1` to the /// highest ID (for a given instrumented function) is valid. /// -/// This visitor runs twice, first with `add_missing_operands` set to `false`, to find the maximum -/// counter ID and maximum expression ID based on their enum variant `id` fields; then, as a -/// safeguard, with `add_missing_operands` set to `true`, to find any other counter or expression -/// IDs referenced by expression operands, if not already seen. -/// -/// Ideally, each operand ID in a MIR `CoverageKind::Expression` will have a separate MIR `Coverage` -/// statement for the `Counter` or `Expression` with the referenced ID. but since current or future -/// MIR optimizations can theoretically optimize out segments of a MIR, it may not be possible to -/// guarantee this, so the second pass ensures the `CoverageInfo` counts include all referenced IDs. +/// It's possible for a coverage expression to remain in MIR while one or both of its operands +/// have been optimized away. To avoid problems in codegen, we include those operands' IDs when +/// determining the maximum counter/expression ID, even if the underlying counter/expression is +/// no longer present. struct CoverageVisitor { - info: CoverageInfo, - add_missing_operands: bool, + max_counter_id: CounterId, + max_expression_id: ExpressionId, } impl CoverageVisitor { - /// Updates `num_counters` to the maximum encountered counter ID plus 1. + /// Updates `max_counter_id` to the maximum encountered counter ID. #[inline(always)] - fn update_num_counters(&mut self, counter_id: CounterId) { - let counter_id = counter_id.as_u32(); - self.info.num_counters = std::cmp::max(self.info.num_counters, counter_id + 1); + fn update_max_counter_id(&mut self, counter_id: CounterId) { + self.max_counter_id = self.max_counter_id.max(counter_id); } - /// Updates `num_expressions` to the maximum encountered expression ID plus 1. + /// Updates `max_expression_id` to the maximum encountered expression ID. #[inline(always)] - fn update_num_expressions(&mut self, expression_id: ExpressionId) { - let expression_id = expression_id.as_u32(); - self.info.num_expressions = std::cmp::max(self.info.num_expressions, expression_id + 1); + fn update_max_expression_id(&mut self, expression_id: ExpressionId) { + self.max_expression_id = self.max_expression_id.max(expression_id); } fn update_from_expression_operand(&mut self, operand: Operand) { match operand { - Operand::Counter(id) => self.update_num_counters(id), - Operand::Expression(id) => self.update_num_expressions(id), + Operand::Counter(id) => self.update_max_counter_id(id), + Operand::Expression(id) => self.update_max_expression_id(id), Operand::Zero => {} } } fn visit_body(&mut self, body: &Body<'_>) { - for bb_data in body.basic_blocks.iter() { - for statement in bb_data.statements.iter() { - if let StatementKind::Coverage(box ref coverage) = statement.kind { - if is_inlined(body, statement) { - continue; - } - self.visit_coverage(coverage); - } - } + for coverage in all_coverage_in_mir_body(body) { + self.visit_coverage(coverage); } } fn visit_coverage(&mut self, coverage: &Coverage) { - if self.add_missing_operands { - match coverage.kind { - CoverageKind::Expression { lhs, rhs, .. } => { - self.update_from_expression_operand(lhs); - self.update_from_expression_operand(rhs); - } - _ => {} - } - } else { - match coverage.kind { - CoverageKind::Counter { id, .. } => self.update_num_counters(id), - CoverageKind::Expression { id, .. } => self.update_num_expressions(id), - _ => {} + match coverage.kind { + CoverageKind::Counter { id, .. } => self.update_max_counter_id(id), + CoverageKind::Expression { id, lhs, rhs, .. } => { + self.update_max_expression_id(id); + self.update_from_expression_operand(lhs); + self.update_from_expression_operand(rhs); } + CoverageKind::Unreachable => {} } } } @@ -101,37 +77,40 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> let mir_body = tcx.instance_mir(instance_def); let mut coverage_visitor = CoverageVisitor { - info: CoverageInfo { num_counters: 0, num_expressions: 0 }, - add_missing_operands: false, + max_counter_id: CounterId::START, + max_expression_id: ExpressionId::START, }; coverage_visitor.visit_body(mir_body); - coverage_visitor.add_missing_operands = true; - coverage_visitor.visit_body(mir_body); - - coverage_visitor.info + // Add 1 to the highest IDs to get the total number of IDs. + CoverageInfo { + num_counters: (coverage_visitor.max_counter_id + 1).as_u32(), + num_expressions: (coverage_visitor.max_expression_id + 1).as_u32(), + } } fn covered_code_regions(tcx: TyCtxt<'_>, def_id: DefId) -> Vec<&CodeRegion> { let body = mir_body(tcx, def_id); - body.basic_blocks - .iter() - .flat_map(|data| { - data.statements.iter().filter_map(|statement| match statement.kind { - StatementKind::Coverage(box ref coverage) => { - if is_inlined(body, statement) { - None - } else { - coverage.code_region.as_ref() // may be None - } - } - _ => None, - }) - }) + all_coverage_in_mir_body(body) + // Not all coverage statements have an attached code region. + .filter_map(|coverage| coverage.code_region.as_ref()) .collect() } +fn all_coverage_in_mir_body<'a, 'tcx>( + body: &'a Body<'tcx>, +) -> impl Iterator<Item = &'a Coverage> + Captures<'tcx> { + body.basic_blocks.iter().flat_map(|bb_data| &bb_data.statements).filter_map(|statement| { + match statement.kind { + StatementKind::Coverage(box ref coverage) if !is_inlined(body, statement) => { + Some(coverage) + } + _ => None, + } + }) +} + fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { let scope_data = &body.source_scopes[statement.source_info.scope]; scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some() diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index d4c9163b571..333a14be996 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -6,10 +6,10 @@ use rustc_const_eval::const_eval::CheckAlignment; use rustc_const_eval::interpret::{ConstValue, ImmTy, Immediate, InterpCx, Scalar}; use rustc_data_structures::fx::FxHashMap; use rustc_hir::def::DefKind; -use rustc_middle::mir::visit::{MutVisitor, Visitor}; +use rustc_middle::mir::visit::{MutVisitor, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{ Map, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace, }; @@ -50,7 +50,7 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp { let place_limit = if tcx.sess.mir_opt_level() < 4 { Some(PLACE_LIMIT) } else { None }; // Decide which places to track during the analysis. - let map = Map::from_filter(tcx, body, Ty::is_scalar, place_limit); + let map = Map::new(tcx, body, place_limit); // Perform the actual dataflow analysis. let analysis = ConstAnalysis::new(tcx, body, map); @@ -58,9 +58,13 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp { .in_scope(|| analysis.wrap().into_engine(tcx, body).iterate_to_fixpoint()); // Collect results and patch the body afterwards. - let mut visitor = CollectAndPatch::new(tcx); + let mut visitor = CollectAndPatch::new(tcx, &body.local_decls); debug_span!("collect").in_scope(|| results.visit_reachable_with(body, &mut visitor)); - debug_span!("patch").in_scope(|| visitor.visit_body(body)); + debug_span!("patch").in_scope(|| { + for (block, bbdata) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + visitor.visit_basic_block_data(block, bbdata); + } + }) } } @@ -73,7 +77,7 @@ struct ConstAnalysis<'a, 'tcx> { } impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { - type Value = FlatSet<ScalarTy<'tcx>>; + type Value = FlatSet<ScalarInt>; const NAME: &'static str = "ConstAnalysis"; @@ -172,9 +176,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { if let Some(overflow_target) = overflow_target { let overflow = match overflow { FlatSet::Top => FlatSet::Top, - FlatSet::Elem(overflow) => { - self.wrap_scalar(Scalar::from_bool(overflow), self.tcx.types.bool) - } + FlatSet::Elem(overflow) => FlatSet::Elem(overflow.into()), FlatSet::Bottom => FlatSet::Bottom, }; // We have flooded `target` earlier. @@ -182,6 +184,23 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { } } } + Rvalue::Cast( + CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize), + operand, + _, + ) => { + let pointer = self.handle_operand(operand, state); + state.assign(target.as_ref(), pointer, self.map()); + + if let Some(target_len) = self.map().find_len(target.as_ref()) + && let operand_ty = operand.ty(self.local_decls, self.tcx) + && let Some(operand_ty) = operand_ty.builtin_deref(true) + && let ty::Array(_, len) = operand_ty.ty.kind() + && let Some(len) = ConstantKind::Ty(*len).eval(self.tcx, self.param_env).try_to_scalar_int() + { + state.insert_value_idx(target_len, FlatSet::Elem(len), self.map()); + } + } _ => self.super_assign(target, rvalue, state), } } @@ -191,47 +210,77 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { rvalue: &Rvalue<'tcx>, state: &mut State<Self::Value>, ) -> ValueOrPlace<Self::Value> { - match rvalue { - Rvalue::Cast( - kind @ (CastKind::IntToInt - | CastKind::FloatToInt - | CastKind::FloatToFloat - | CastKind::IntToFloat), - operand, - ty, - ) => match self.eval_operand(operand, state) { - FlatSet::Elem(op) => match kind { - CastKind::IntToInt | CastKind::IntToFloat => { - self.ecx.int_to_int_or_float(&op, *ty) - } - CastKind::FloatToInt | CastKind::FloatToFloat => { - self.ecx.float_to_float_or_int(&op, *ty) - } - _ => unreachable!(), + let val = match rvalue { + Rvalue::Len(place) => { + let place_ty = place.ty(self.local_decls, self.tcx); + if let ty::Array(_, len) = place_ty.ty.kind() { + ConstantKind::Ty(*len) + .eval(self.tcx, self.param_env) + .try_to_scalar_int() + .map_or(FlatSet::Top, FlatSet::Elem) + } else if let [ProjectionElem::Deref] = place.projection[..] { + state.get_len(place.local.into(), self.map()) + } else { + FlatSet::Top } - .map(|result| ValueOrPlace::Value(self.wrap_immediate(result, *ty))) - .unwrap_or(ValueOrPlace::TOP), - _ => ValueOrPlace::TOP, - }, + } + Rvalue::Cast(CastKind::IntToInt | CastKind::IntToFloat, operand, ty) => { + match self.eval_operand(operand, state) { + FlatSet::Elem(op) => self + .ecx + .int_to_int_or_float(&op, *ty) + .map_or(FlatSet::Top, |result| self.wrap_immediate(result)), + FlatSet::Bottom => FlatSet::Bottom, + FlatSet::Top => FlatSet::Top, + } + } + Rvalue::Cast(CastKind::FloatToInt | CastKind::FloatToFloat, operand, ty) => { + match self.eval_operand(operand, state) { + FlatSet::Elem(op) => self + .ecx + .float_to_float_or_int(&op, *ty) + .map_or(FlatSet::Top, |result| self.wrap_immediate(result)), + FlatSet::Bottom => FlatSet::Bottom, + FlatSet::Top => FlatSet::Top, + } + } + Rvalue::Cast(CastKind::Transmute, operand, _) => { + match self.eval_operand(operand, state) { + FlatSet::Elem(op) => self.wrap_immediate(*op), + FlatSet::Bottom => FlatSet::Bottom, + FlatSet::Top => FlatSet::Top, + } + } Rvalue::BinaryOp(op, box (left, right)) => { // Overflows must be ignored here. let (val, _overflow) = self.binary_op(state, *op, left, right); - ValueOrPlace::Value(val) + val } Rvalue::UnaryOp(op, operand) => match self.eval_operand(operand, state) { - FlatSet::Elem(value) => self - .ecx - .unary_op(*op, &value) - .map(|val| ValueOrPlace::Value(self.wrap_immty(val))) - .unwrap_or(ValueOrPlace::Value(FlatSet::Top)), - FlatSet::Bottom => ValueOrPlace::Value(FlatSet::Bottom), - FlatSet::Top => ValueOrPlace::Value(FlatSet::Top), + FlatSet::Elem(value) => { + self.ecx.unary_op(*op, &value).map_or(FlatSet::Top, |val| self.wrap_immty(val)) + } + FlatSet::Bottom => FlatSet::Bottom, + FlatSet::Top => FlatSet::Top, }, - Rvalue::Discriminant(place) => { - ValueOrPlace::Value(state.get_discr(place.as_ref(), self.map())) + Rvalue::NullaryOp(null_op, ty) => { + let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + return ValueOrPlace::Value(FlatSet::Top); + }; + let val = match null_op { + NullOp::SizeOf if layout.is_sized() => layout.size.bytes(), + NullOp::AlignOf if layout.is_sized() => layout.align.abi.bytes(), + NullOp::OffsetOf(fields) => layout + .offset_of_subfield(&self.ecx, fields.iter().map(|f| f.index())) + .bytes(), + _ => return ValueOrPlace::Value(FlatSet::Top), + }; + ScalarInt::try_from_target_usize(val, self.tcx).map_or(FlatSet::Top, FlatSet::Elem) } - _ => self.super_rvalue(rvalue, state), - } + Rvalue::Discriminant(place) => state.get_discr(place.as_ref(), self.map()), + _ => return self.super_rvalue(rvalue, state), + }; + ValueOrPlace::Value(val) } fn handle_constant( @@ -242,9 +291,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { constant .literal .eval(self.tcx, self.param_env) - .try_to_scalar() - .map(|value| FlatSet::Elem(ScalarTy(value, constant.ty()))) - .unwrap_or(FlatSet::Top) + .try_to_scalar_int() + .map_or(FlatSet::Top, FlatSet::Elem) } fn handle_switch_int<'mir>( @@ -261,9 +309,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { // We are branching on uninitialized data, this is UB, treat it as unreachable. // This allows the set of visited edges to grow monotonically with the lattice. FlatSet::Bottom => TerminatorEdges::None, - FlatSet::Elem(ScalarTy(scalar, _)) => { - let int = scalar.assert_int(); - let choice = int.assert_bits(int.size()); + FlatSet::Elem(scalar) => { + let choice = scalar.assert_bits(scalar.size()); TerminatorEdges::Single(targets.target_for_value(choice)) } FlatSet::Top => TerminatorEdges::SwitchInt { discr, targets }, @@ -271,16 +318,6 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { } } -#[derive(Clone, PartialEq, Eq)] -struct ScalarTy<'tcx>(Scalar, Ty<'tcx>); - -impl<'tcx> std::fmt::Debug for ScalarTy<'tcx> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // This is used for dataflow visualization, so we return something more concise. - std::fmt::Display::fmt(&ConstantKind::Val(ConstValue::Scalar(self.0), self.1), f) - } -} - impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map) -> Self { let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); @@ -295,32 +332,62 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { fn binary_op( &self, - state: &mut State<FlatSet<ScalarTy<'tcx>>>, + state: &mut State<FlatSet<ScalarInt>>, op: BinOp, left: &Operand<'tcx>, right: &Operand<'tcx>, - ) -> (FlatSet<ScalarTy<'tcx>>, FlatSet<bool>) { + ) -> (FlatSet<ScalarInt>, FlatSet<bool>) { let left = self.eval_operand(left, state); let right = self.eval_operand(right, state); + match (left, right) { + (FlatSet::Bottom, _) | (_, FlatSet::Bottom) => (FlatSet::Bottom, FlatSet::Bottom), + // Both sides are known, do the actual computation. (FlatSet::Elem(left), FlatSet::Elem(right)) => { match self.ecx.overflowing_binary_op(op, &left, &right) { - Ok((val, overflow, ty)) => (self.wrap_scalar(val, ty), FlatSet::Elem(overflow)), + Ok((Scalar::Int(val), overflow, _)) => { + (FlatSet::Elem(val), FlatSet::Elem(overflow)) + } _ => (FlatSet::Top, FlatSet::Top), } } - (FlatSet::Bottom, _) | (_, FlatSet::Bottom) => (FlatSet::Bottom, FlatSet::Bottom), - (_, _) => { - // Could attempt some algebraic simplifications here. - (FlatSet::Top, FlatSet::Top) + // Exactly one side is known, attempt some algebraic simplifications. + (FlatSet::Elem(const_arg), _) | (_, FlatSet::Elem(const_arg)) => { + let layout = const_arg.layout; + if !matches!(layout.abi, rustc_target::abi::Abi::Scalar(..)) { + return (FlatSet::Top, FlatSet::Top); + } + + let arg_scalar = const_arg.to_scalar(); + let Ok(arg_scalar) = arg_scalar.try_to_int() else { + return (FlatSet::Top, FlatSet::Top); + }; + let Ok(arg_value) = arg_scalar.to_bits(layout.size) else { + return (FlatSet::Top, FlatSet::Top); + }; + + match op { + BinOp::BitAnd if arg_value == 0 => (FlatSet::Elem(arg_scalar), FlatSet::Bottom), + BinOp::BitOr + if arg_value == layout.size.truncate(u128::MAX) + || (layout.ty.is_bool() && arg_value == 1) => + { + (FlatSet::Elem(arg_scalar), FlatSet::Bottom) + } + BinOp::Mul if layout.ty.is_integral() && arg_value == 0 => { + (FlatSet::Elem(arg_scalar), FlatSet::Elem(false)) + } + _ => (FlatSet::Top, FlatSet::Top), + } } + (FlatSet::Top, FlatSet::Top) => (FlatSet::Top, FlatSet::Top), } } fn eval_operand( &self, op: &Operand<'tcx>, - state: &mut State<FlatSet<ScalarTy<'tcx>>>, + state: &mut State<FlatSet<ScalarInt>>, ) -> FlatSet<ImmTy<'tcx>> { let value = match self.handle_operand(op, state) { ValueOrPlace::Value(value) => value, @@ -328,76 +395,76 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { }; match value { FlatSet::Top => FlatSet::Top, - FlatSet::Elem(ScalarTy(scalar, ty)) => self - .tcx - .layout_of(self.param_env.and(ty)) - .map(|layout| FlatSet::Elem(ImmTy::from_scalar(scalar, layout))) - .unwrap_or(FlatSet::Top), + FlatSet::Elem(scalar) => { + let ty = op.ty(self.local_decls, self.tcx); + self.tcx + .layout_of(self.param_env.and(ty)) + .map(|layout| FlatSet::Elem(ImmTy::from_scalar(scalar.into(), layout))) + .unwrap_or(FlatSet::Top) + } FlatSet::Bottom => FlatSet::Bottom, } } - fn eval_discriminant( - &self, - enum_ty: Ty<'tcx>, - variant_index: VariantIdx, - ) -> Option<ScalarTy<'tcx>> { + fn eval_discriminant(&self, enum_ty: Ty<'tcx>, variant_index: VariantIdx) -> Option<ScalarInt> { if !enum_ty.is_enum() { return None; } let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?; let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?; - let discr_value = Scalar::try_from_uint(discr.val, discr_layout.size)?; - Some(ScalarTy(discr_value, discr.ty)) - } - - fn wrap_scalar(&self, scalar: Scalar, ty: Ty<'tcx>) -> FlatSet<ScalarTy<'tcx>> { - FlatSet::Elem(ScalarTy(scalar, ty)) + let discr_value = ScalarInt::try_from_uint(discr.val, discr_layout.size)?; + Some(discr_value) } - fn wrap_immediate(&self, imm: Immediate, ty: Ty<'tcx>) -> FlatSet<ScalarTy<'tcx>> { + fn wrap_immediate(&self, imm: Immediate) -> FlatSet<ScalarInt> { match imm { - Immediate::Scalar(scalar) => self.wrap_scalar(scalar, ty), + Immediate::Scalar(Scalar::Int(scalar)) => FlatSet::Elem(scalar), _ => FlatSet::Top, } } - fn wrap_immty(&self, val: ImmTy<'tcx>) -> FlatSet<ScalarTy<'tcx>> { - self.wrap_immediate(*val, val.layout.ty) + fn wrap_immty(&self, val: ImmTy<'tcx>) -> FlatSet<ScalarInt> { + self.wrap_immediate(*val) } } -struct CollectAndPatch<'tcx> { +struct CollectAndPatch<'tcx, 'locals> { tcx: TyCtxt<'tcx>, + local_decls: &'locals LocalDecls<'tcx>, /// For a given MIR location, this stores the values of the operands used by that location. In /// particular, this is before the effect, such that the operands of `_1 = _1 + _2` are /// properly captured. (This may become UB soon, but it is currently emitted even by safe code.) - before_effect: FxHashMap<(Location, Place<'tcx>), ScalarTy<'tcx>>, + before_effect: FxHashMap<(Location, Place<'tcx>), ScalarInt>, /// Stores the assigned values for assignments where the Rvalue is constant. - assignments: FxHashMap<Location, ScalarTy<'tcx>>, + assignments: FxHashMap<Location, ScalarInt>, } -impl<'tcx> CollectAndPatch<'tcx> { - fn new(tcx: TyCtxt<'tcx>) -> Self { - Self { tcx, before_effect: FxHashMap::default(), assignments: FxHashMap::default() } +impl<'tcx, 'locals> CollectAndPatch<'tcx, 'locals> { + fn new(tcx: TyCtxt<'tcx>, local_decls: &'locals LocalDecls<'tcx>) -> Self { + Self { + tcx, + local_decls, + before_effect: FxHashMap::default(), + assignments: FxHashMap::default(), + } } - fn make_operand(&self, scalar: ScalarTy<'tcx>) -> Operand<'tcx> { + fn make_operand(&self, scalar: ScalarInt, ty: Ty<'tcx>) -> Operand<'tcx> { Operand::Constant(Box::new(Constant { span: DUMMY_SP, user_ty: None, - literal: ConstantKind::Val(ConstValue::Scalar(scalar.0), scalar.1), + literal: ConstantKind::Val(ConstValue::Scalar(scalar.into()), ty), })) } } impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>> - for CollectAndPatch<'tcx> + for CollectAndPatch<'tcx, '_> { - type FlowState = State<FlatSet<ScalarTy<'tcx>>>; + type FlowState = State<FlatSet<ScalarInt>>; fn visit_statement_before_primary_effect( &mut self, @@ -453,8 +520,8 @@ impl<'mir, 'tcx> } } -impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx> { - fn tcx<'a>(&'a self) -> TyCtxt<'tcx> { +impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { + fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -462,7 +529,8 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx> { if let Some(value) = self.assignments.get(&location) { match &mut statement.kind { StatementKind::Assign(box (_, rvalue)) => { - *rvalue = Rvalue::Use(self.make_operand(value.clone())); + let ty = rvalue.ty(self.local_decls, self.tcx); + *rvalue = Rvalue::Use(self.make_operand(*value, ty)); } _ => bug!("found assignment info for non-assign statement"), } @@ -475,33 +543,56 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx> { match operand { Operand::Copy(place) | Operand::Move(place) => { if let Some(value) = self.before_effect.get(&(location, *place)) { - *operand = self.make_operand(value.clone()); + let ty = place.ty(self.local_decls, self.tcx).ty; + *operand = self.make_operand(*value, ty); + } else if !place.projection.is_empty() { + self.super_operand(operand, location) } } - _ => (), + Operand::Constant(_) => {} + } + } + + fn process_projection_elem( + &mut self, + elem: PlaceElem<'tcx>, + location: Location, + ) -> Option<PlaceElem<'tcx>> { + if let PlaceElem::Index(local) = elem + && let Some(value) = self.before_effect.get(&(location, local.into())) + && let Ok(offset) = value.try_to_target_usize(self.tcx) + && let Some(min_length) = offset.checked_add(1) + { + Some(PlaceElem::ConstantIndex { offset, min_length, from_end: false }) + } else { + None } } } -struct OperandCollector<'tcx, 'map, 'a> { - state: &'a State<FlatSet<ScalarTy<'tcx>>>, - visitor: &'a mut CollectAndPatch<'tcx>, +struct OperandCollector<'tcx, 'map, 'locals, 'a> { + state: &'a State<FlatSet<ScalarInt>>, + visitor: &'a mut CollectAndPatch<'tcx, 'locals>, map: &'map Map, } -impl<'tcx, 'map, 'a> Visitor<'tcx> for OperandCollector<'tcx, 'map, 'a> { +impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> { fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { - match operand { - Operand::Copy(place) | Operand::Move(place) => { - match self.state.get(place.as_ref(), self.map) { - FlatSet::Top => (), - FlatSet::Elem(value) => { - self.visitor.before_effect.insert((location, *place), value); - } - FlatSet::Bottom => (), - } + if let Some(place) = operand.place() { + if let FlatSet::Elem(value) = self.state.get(place.as_ref(), self.map) { + self.visitor.before_effect.insert((location, place), value); + } else if !place.projection.is_empty() { + // Try to propagate into `Index` projections. + self.super_operand(operand, location) } - _ => (), + } + } + + fn visit_local(&mut self, local: Local, ctxt: PlaceContext, location: Location) { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy | NonMutatingUseContext::Move) = ctxt + && let FlatSet::Elem(value) = self.state.get(local.into(), self.map) + { + self.visitor.before_effect.insert((location, local.into()), value); } } } @@ -572,7 +663,7 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm _bin_op: BinOp, _left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>, _right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>, - ) -> interpret::InterpResult<'tcx, (interpret::Scalar<Self::Provenance>, bool, Ty<'tcx>)> { + ) -> interpret::InterpResult<'tcx, (Scalar<Self::Provenance>, bool, Ty<'tcx>)> { throw_unsup!(Unsupported("".into())) } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 4b796d79ef6..35373bcaa41 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -258,10 +258,3 @@ pub(crate) struct MustNotSuspendReason { pub span: Span, pub reason: String, } - -#[derive(Diagnostic)] -#[diag(mir_transform_simd_shuffle_last_const)] -pub(crate) struct SimdShuffleLastConst { - #[primary_span] - pub span: Span, -} diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index a8dc91ca439..13277d62bf4 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -1,11 +1,10 @@ //! Lowers intrinsic calls -use crate::{errors, MirPass}; +use crate::MirPass; use rustc_middle::mir::*; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; -use rustc_span::Span; use rustc_target::abi::{FieldIdx, VariantIdx}; pub struct LowerIntrinsics; @@ -304,9 +303,6 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Unreachable; } } - sym::simd_shuffle => { - validate_simd_shuffle(tcx, args, terminator.source_info.span); - } _ => {} } } @@ -325,9 +321,3 @@ fn resolve_rust_intrinsic<'tcx>( } None } - -fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { - if !matches!(args[2], Operand::Constant(_)) { - tcx.sess.emit_err(errors::SimdShuffleLastConst { span }); - } -} diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index 9c6c55b0811..c13bafa9fbb 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -87,11 +87,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { var_debug_info.value = VarDebugInfoContents::Const(self.make_zst(place_ty)) } } - VarDebugInfoContents::Composite { ty, fragments: _ } => { - if self.known_to_be_zst(ty) { - var_debug_info.value = VarDebugInfoContents::Const(self.make_zst(ty)) - } - } } } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index e66ae8ff884..c21b1724cbb 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -1,4 +1,5 @@ use crate::MirPass; +use rustc_data_structures::flat_map_in_place::FlatMapInPlace; use rustc_index::bit_set::{BitSet, GrowableBitSet}; use rustc_index::IndexVec; use rustc_middle::mir::patch::MirPatch; @@ -147,7 +148,7 @@ fn escaping_locals<'tcx>( } // We ignore anything that happens in debuginfo, since we expand it using - // `VarDebugInfoContents::Composite`. + // `VarDebugInfoFragment`. fn visit_var_debug_info(&mut self, _: &VarDebugInfo<'tcx>) {} } } @@ -246,9 +247,7 @@ fn replace_flattened_locals<'tcx>( for (index, annotation) in body.user_type_annotations.iter_enumerated_mut() { visitor.visit_user_type_annotation(index, annotation); } - for var_debug_info in &mut body.var_debug_info { - visitor.visit_var_debug_info(var_debug_info); - } + visitor.expand_var_debug_info(&mut body.var_debug_info); let ReplacementVisitor { patch, all_dead_locals, .. } = visitor; patch.apply(body); all_dead_locals @@ -256,7 +255,7 @@ fn replace_flattened_locals<'tcx>( struct ReplacementVisitor<'tcx, 'll> { tcx: TyCtxt<'tcx>, - /// This is only used to compute the type for `VarDebugInfoContents::Composite`. + /// This is only used to compute the type for `VarDebugInfoFragment`. local_decls: &'ll LocalDecls<'tcx>, /// Work to do. replacements: &'ll ReplacementMap<'tcx>, @@ -266,16 +265,38 @@ struct ReplacementVisitor<'tcx, 'll> { } impl<'tcx> ReplacementVisitor<'tcx, '_> { - fn gather_debug_info_fragments(&self, local: Local) -> Option<Vec<VarDebugInfoFragment<'tcx>>> { - let mut fragments = Vec::new(); - let parts = self.replacements.place_fragments(local.into())?; - for (field, ty, replacement_local) in parts { - fragments.push(VarDebugInfoFragment { - projection: vec![PlaceElem::Field(field, ty)], - contents: Place::from(replacement_local), - }); - } - Some(fragments) + #[instrument(level = "trace", skip(self))] + fn expand_var_debug_info(&mut self, var_debug_info: &mut Vec<VarDebugInfo<'tcx>>) { + var_debug_info.flat_map_in_place(|mut var_debug_info| { + let place = match var_debug_info.value { + VarDebugInfoContents::Const(_) => return vec![var_debug_info], + VarDebugInfoContents::Place(ref mut place) => place, + }; + + if let Some(repl) = self.replacements.replace_place(self.tcx, place.as_ref()) { + *place = repl; + return vec![var_debug_info]; + } + + let Some(parts) = self.replacements.place_fragments(*place) else { + return vec![var_debug_info]; + }; + + let ty = place.ty(self.local_decls, self.tcx).ty; + + parts + .map(|(field, field_ty, replacement_local)| { + let mut var_debug_info = var_debug_info.clone(); + let composite = var_debug_info.composite.get_or_insert_with(|| { + Box::new(VarDebugInfoFragment { ty, projection: Vec::new() }) + }); + composite.projection.push(PlaceElem::Field(field, field_ty)); + + var_debug_info.value = VarDebugInfoContents::Place(replacement_local.into()); + var_debug_info + }) + .collect() + }); } } @@ -422,48 +443,6 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { self.super_statement(statement, location) } - #[instrument(level = "trace", skip(self))] - fn visit_var_debug_info(&mut self, var_debug_info: &mut VarDebugInfo<'tcx>) { - match &mut var_debug_info.value { - VarDebugInfoContents::Place(ref mut place) => { - if let Some(repl) = self.replacements.replace_place(self.tcx, place.as_ref()) { - *place = repl; - } else if let Some(local) = place.as_local() - && let Some(fragments) = self.gather_debug_info_fragments(local) - { - let ty = place.ty(self.local_decls, self.tcx).ty; - var_debug_info.value = VarDebugInfoContents::Composite { ty, fragments }; - } - } - VarDebugInfoContents::Composite { ty: _, ref mut fragments } => { - let mut new_fragments = Vec::new(); - debug!(?fragments); - fragments.retain_mut(|fragment| { - if let Some(repl) = - self.replacements.replace_place(self.tcx, fragment.contents.as_ref()) - { - fragment.contents = repl; - true - } else if let Some(local) = fragment.contents.as_local() - && let Some(frg) = self.gather_debug_info_fragments(local) - { - new_fragments.extend(frg.into_iter().map(|mut f| { - f.projection.splice(0..0, fragment.projection.iter().copied()); - f - })); - false - } else { - true - } - }); - debug!(?fragments); - debug!(?new_fragments); - fragments.extend(new_fragments); - } - VarDebugInfoContents::Const(_) => {} - } - } - fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) { assert!(!self.all_dead_locals.contains(*local)); } diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 5d6c574baa6..c4e8d9006e6 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -106,7 +106,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { let mut cursor_snapshot = self.cursor_snapshot.clone(); let tokens = std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain((0..self.num_calls).map(|_| { + .chain(std::iter::repeat_with(|| { let token = cursor_snapshot.next(); (FlatToken::Token(token.0), token.1) })) diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 233c7016417..aad4edaba90 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -73,12 +73,16 @@ impl<'a> Parser<'a> { if !self.maybe_consume_incorrect_semicolon(&items) { let msg = format!("expected item, found {token_str}"); let mut err = self.struct_span_err(self.token.span, msg); - let label = if self.is_kw_followed_by_ident(kw::Let) { - "consider using `const` or `static` instead of `let` for global variables" + let span = self.token.span; + if self.is_kw_followed_by_ident(kw::Let) { + err.span_label( + span, + "consider using `const` or `static` instead of `let` for global variables", + ); } else { - "expected item" + err.span_label(span, "expected item") + .note("for a full list of items that can appear in modules, see <https://doc.rust-lang.org/reference/items.html>"); }; - err.span_label(self.token.span, label); return Err(err); } } diff --git a/compiler/rustc_parse_format/Cargo.toml b/compiler/rustc_parse_format/Cargo.toml index 72da398d3fc..14330353239 100644 --- a/compiler/rustc_parse_format/Cargo.toml +++ b/compiler/rustc_parse_format/Cargo.toml @@ -5,4 +5,4 @@ edition = "2021" [dependencies] rustc_lexer = { path = "../rustc_lexer" } -rustc_data_structures = { path = "../rustc_data_structures" } +rustc_index = { path = "../rustc_index", default-features = false } diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs index 88452ccdf05..a6c7eb8d912 100644 --- a/compiler/rustc_parse_format/src/lib.rs +++ b/compiler/rustc_parse_format/src/lib.rs @@ -1011,7 +1011,7 @@ fn unescape_string(string: &str) -> Option<string::String> { // Assert a reasonable size for `Piece` #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(Piece<'_>, 16); +rustc_index::static_assert_size!(Piece<'_>, 16); #[cfg(test)] mod tests; diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl index 57598cf8bcf..9f3cd656bd1 100644 --- a/compiler/rustc_passes/messages.ftl +++ b/compiler/rustc_passes/messages.ftl @@ -4,14 +4,14 @@ -passes_see_issue = see issue #{$issue} <https://github.com/rust-lang/rust/issues/{$issue}> for more information -passes_abi = - abi: {$abi} - +passes_abi_invalid_attribute = + `#[rustc_abi]` can only be applied to function items, type aliases, and associated functions +passes_abi_ne = + ABIs are not compatible + left ABI = {$left} + right ABI = {$right} passes_abi_of = - fn_abi_of_instance({$fn_name}) = {$fn_abi} - -passes_align = - align: {$align} + fn_abi_of({$fn_name}) = {$fn_abi} passes_allow_incoherent_impl = `rustc_allow_incoherent_impl` attribute should be applied to impl items. @@ -318,9 +318,6 @@ passes_has_incoherent_inherent_impl = `rustc_has_incoherent_inherent_impls` attribute should be applied to types or traits. .label = only adts, extern types and traits are supported -passes_homogeneous_aggregate = - homogeneous_aggregate: {$homogeneous_aggregate} - passes_ignored_attr = `#[{$sym}]` is ignored on struct fields and match arms .warn = {-passes_previously_accepted} @@ -404,9 +401,18 @@ passes_lang_item_on_incorrect_target = passes_layout = layout error: {$layout_error} - +passes_layout_abi = + abi: {$abi} +passes_layout_align = + align: {$align} +passes_layout_homogeneous_aggregate = + homogeneous_aggregate: {$homogeneous_aggregate} +passes_layout_invalid_attribute = + `#[rustc_layout]` can only be applied to `struct`/`enum`/`union` declarations and type aliases passes_layout_of = layout_of({$normalized_ty}) = {$ty_layout} +passes_layout_size = + size: {$size} passes_link = attribute should be applied to an `extern` block with non-Rust ABI @@ -662,9 +668,6 @@ passes_should_be_applied_to_trait = attribute should be applied to a trait .label = not a trait -passes_size = - size: {$size} - passes_skipping_const_checks = skipping const checks passes_stability_promotable = diff --git a/compiler/rustc_passes/src/abi_test.rs b/compiler/rustc_passes/src/abi_test.rs index 5c0438e78ae..7e781bdd6f4 100644 --- a/compiler/rustc_passes/src/abi_test.rs +++ b/compiler/rustc_passes/src/abi_test.rs @@ -2,11 +2,12 @@ use rustc_ast::Attribute; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_middle::ty::layout::{FnAbiError, LayoutError}; -use rustc_middle::ty::{self, GenericArgs, Instance, TyCtxt}; +use rustc_middle::ty::{self, GenericArgs, Instance, Ty, TyCtxt}; use rustc_span::source_map::Spanned; use rustc_span::symbol::sym; +use rustc_target::abi::call::FnAbi; -use crate::errors::{AbiOf, UnrecognizedField}; +use crate::errors::{AbiInvalidAttribute, AbiNe, AbiOf, UnrecognizedField}; pub fn test_abi(tcx: TyCtxt<'_>) { if !tcx.features().rustc_attrs { @@ -14,28 +15,63 @@ pub fn test_abi(tcx: TyCtxt<'_>) { return; } for id in tcx.hir().items() { - match tcx.def_kind(id.owner_id) { - DefKind::Fn => { - for attr in tcx.get_attrs(id.owner_id, sym::rustc_abi) { - dump_abi_of(tcx, id.owner_id.def_id.into(), attr); + for attr in tcx.get_attrs(id.owner_id, sym::rustc_abi) { + match tcx.def_kind(id.owner_id) { + DefKind::Fn => { + dump_abi_of_fn_item(tcx, id.owner_id.def_id.into(), attr); + } + DefKind::TyAlias { .. } => { + dump_abi_of_fn_type(tcx, id.owner_id.def_id.into(), attr); + } + _ => { + tcx.sess.emit_err(AbiInvalidAttribute { span: tcx.def_span(id.owner_id) }); } } - DefKind::Impl { .. } => { - // To find associated functions we need to go into the child items here. - for &id in tcx.associated_item_def_ids(id.owner_id) { - if matches!(tcx.def_kind(id), DefKind::AssocFn) { - for attr in tcx.get_attrs(id, sym::rustc_abi) { - dump_abi_of(tcx, id, attr); + } + if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. }) { + // To find associated functions we need to go into the child items here. + for &id in tcx.associated_item_def_ids(id.owner_id) { + for attr in tcx.get_attrs(id, sym::rustc_abi) { + match tcx.def_kind(id) { + DefKind::AssocFn => { + dump_abi_of_fn_item(tcx, id, attr); + } + _ => { + tcx.sess.emit_err(AbiInvalidAttribute { span: tcx.def_span(id) }); } } } } - _ => {} } } } -fn dump_abi_of(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) { +fn unwrap_fn_abi<'tcx>( + abi: Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, &'tcx FnAbiError<'tcx>>, + tcx: TyCtxt<'tcx>, + item_def_id: DefId, +) -> &'tcx FnAbi<'tcx, Ty<'tcx>> { + match abi { + Ok(abi) => abi, + Err(FnAbiError::Layout(layout_error)) => { + tcx.sess.emit_fatal(Spanned { + node: layout_error.into_diagnostic(), + span: tcx.def_span(item_def_id), + }); + } + Err(FnAbiError::AdjustForForeignAbi(e)) => { + // Sadly there seems to be no `into_diagnostic` for this case... and I am not sure if + // this can even be reached. Anyway this is a perma-unstable debug attribute, an ICE + // isn't the worst thing. Also this matches what codegen does. + span_bug!( + tcx.def_span(item_def_id), + "error computing fn_abi_of_instance, cannot adjust for foreign ABI: {e:?}", + ) + } + } +} + +fn dump_abi_of_fn_item(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) { let param_env = tcx.param_env(item_def_id); let args = GenericArgs::identity_for_item(tcx, item_def_id); let instance = match Instance::resolve(tcx, param_env, item_def_id, args) { @@ -51,43 +87,124 @@ fn dump_abi_of(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) { } Err(_guaranteed) => return, }; - match tcx.fn_abi_of_instance(param_env.and((instance, /* extra_args */ ty::List::empty()))) { - Ok(abi) => { - // Check out the `#[rustc_abi(..)]` attribute to tell what to dump. - // The `..` are the names of fields to dump. - let meta_items = attr.meta_item_list().unwrap_or_default(); - for meta_item in meta_items { - match meta_item.name_or_empty() { - sym::debug => { - let fn_name = tcx.item_name(item_def_id); - tcx.sess.emit_err(AbiOf { - span: tcx.def_span(item_def_id), - fn_name, - fn_abi: format!("{:#?}", abi), - }); - } + let abi = unwrap_fn_abi( + tcx.fn_abi_of_instance(param_env.and((instance, /* extra_args */ ty::List::empty()))), + tcx, + item_def_id, + ); - name => { - tcx.sess.emit_err(UnrecognizedField { span: meta_item.span(), name }); - } - } + // Check out the `#[rustc_abi(..)]` attribute to tell what to dump. + // The `..` are the names of fields to dump. + let meta_items = attr.meta_item_list().unwrap_or_default(); + for meta_item in meta_items { + match meta_item.name_or_empty() { + sym::debug => { + let fn_name = tcx.item_name(item_def_id); + tcx.sess.emit_err(AbiOf { + span: tcx.def_span(item_def_id), + fn_name, + fn_abi: format!("{:#?}", abi), + }); } - } - Err(FnAbiError::Layout(layout_error)) => { - tcx.sess.emit_fatal(Spanned { - node: layout_error.into_diagnostic(), - span: tcx.def_span(item_def_id), - }); + name => { + tcx.sess.emit_err(UnrecognizedField { span: meta_item.span(), name }); + } } - Err(FnAbiError::AdjustForForeignAbi(e)) => { - // Sadly there seems to be no `into_diagnostic` for this case... and I am not sure if - // this can even be reached. Anyway this is a perma-unstable debug attribute, an ICE - // isn't the worst thing. Also this matches what codegen does. - span_bug!( - tcx.def_span(item_def_id), - "error computing fn_abi_of_instance, cannot adjust for foreign ABI: {e:?}", - ) + } +} + +fn test_abi_eq<'tcx>(abi1: &'tcx FnAbi<'tcx, Ty<'tcx>>, abi2: &'tcx FnAbi<'tcx, Ty<'tcx>>) -> bool { + if abi1.conv != abi2.conv + || abi1.args.len() != abi2.args.len() + || abi1.c_variadic != abi2.c_variadic + || abi1.fixed_count != abi2.fixed_count + || abi1.can_unwind != abi2.can_unwind + { + return false; + } + + abi1.ret.eq_abi(&abi2.ret) + && abi1.args.iter().zip(abi2.args.iter()).all(|(arg1, arg2)| arg1.eq_abi(arg2)) +} + +fn dump_abi_of_fn_type(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) { + let param_env = tcx.param_env(item_def_id); + let ty = tcx.type_of(item_def_id).instantiate_identity(); + let meta_items = attr.meta_item_list().unwrap_or_default(); + for meta_item in meta_items { + match meta_item.name_or_empty() { + sym::debug => { + let ty::FnPtr(sig) = ty.kind() else { + span_bug!( + meta_item.span(), + "`#[rustc_abi(debug)]` on a type alias requires function pointer type" + ); + }; + let abi = unwrap_fn_abi( + tcx.fn_abi_of_fn_ptr(param_env.and((*sig, /* extra_args */ ty::List::empty()))), + tcx, + item_def_id, + ); + + let fn_name = tcx.item_name(item_def_id); + tcx.sess.emit_err(AbiOf { + span: tcx.def_span(item_def_id), + fn_name, + fn_abi: format!("{:#?}", abi), + }); + } + sym::assert_eq => { + let ty::Tuple(fields) = ty.kind() else { + span_bug!( + meta_item.span(), + "`#[rustc_abi(assert_eq)]` on a type alias requires pair type" + ); + }; + let [field1, field2] = ***fields else { + span_bug!( + meta_item.span(), + "`#[rustc_abi(assert_eq)]` on a type alias requires pair type" + ); + }; + let ty::FnPtr(sig1) = field1.kind() else { + span_bug!( + meta_item.span(), + "`#[rustc_abi(assert_eq)]` on a type alias requires pair of function pointer types" + ); + }; + let abi1 = unwrap_fn_abi( + tcx.fn_abi_of_fn_ptr( + param_env.and((*sig1, /* extra_args */ ty::List::empty())), + ), + tcx, + item_def_id, + ); + let ty::FnPtr(sig2) = field2.kind() else { + span_bug!( + meta_item.span(), + "`#[rustc_abi(assert_eq)]` on a type alias requires pair of function pointer types" + ); + }; + let abi2 = unwrap_fn_abi( + tcx.fn_abi_of_fn_ptr( + param_env.and((*sig2, /* extra_args */ ty::List::empty())), + ), + tcx, + item_def_id, + ); + + if !test_abi_eq(abi1, abi2) { + tcx.sess.emit_err(AbiNe { + span: tcx.def_span(item_def_id), + left: format!("{:#?}", abi1), + right: format!("{:#?}", abi2), + }); + } + } + name => { + tcx.sess.emit_err(UnrecognizedField { span: meta_item.span(), name }); + } } } } diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs index 32dd02a4aa9..d7319ef91e0 100644 --- a/compiler/rustc_passes/src/errors.rs +++ b/compiler/rustc_passes/src/errors.rs @@ -873,32 +873,32 @@ pub struct DuplicateDiagnosticItemInCrate { } #[derive(Diagnostic)] -#[diag(passes_abi)] -pub struct Abi { +#[diag(passes_layout_abi)] +pub struct LayoutAbi { #[primary_span] pub span: Span, pub abi: String, } #[derive(Diagnostic)] -#[diag(passes_align)] -pub struct Align { +#[diag(passes_layout_align)] +pub struct LayoutAlign { #[primary_span] pub span: Span, pub align: String, } #[derive(Diagnostic)] -#[diag(passes_size)] -pub struct Size { +#[diag(passes_layout_size)] +pub struct LayoutSize { #[primary_span] pub span: Span, pub size: String, } #[derive(Diagnostic)] -#[diag(passes_homogeneous_aggregate)] -pub struct HomogeneousAggregate { +#[diag(passes_layout_homogeneous_aggregate)] +pub struct LayoutHomogeneousAggregate { #[primary_span] pub span: Span, pub homogeneous_aggregate: String, @@ -914,6 +914,13 @@ pub struct LayoutOf { } #[derive(Diagnostic)] +#[diag(passes_layout_invalid_attribute)] +pub struct LayoutInvalidAttribute { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] #[diag(passes_abi_of)] pub struct AbiOf { #[primary_span] @@ -923,6 +930,22 @@ pub struct AbiOf { } #[derive(Diagnostic)] +#[diag(passes_abi_ne)] +pub struct AbiNe { + #[primary_span] + pub span: Span, + pub left: String, + pub right: String, +} + +#[derive(Diagnostic)] +#[diag(passes_abi_invalid_attribute)] +pub struct AbiInvalidAttribute { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] #[diag(passes_unrecognized_field)] pub struct UnrecognizedField { #[primary_span] diff --git a/compiler/rustc_passes/src/layout_test.rs b/compiler/rustc_passes/src/layout_test.rs index d839fee07a6..f3c12e0746d 100644 --- a/compiler/rustc_passes/src/layout_test.rs +++ b/compiler/rustc_passes/src/layout_test.rs @@ -8,7 +8,10 @@ use rustc_span::symbol::sym; use rustc_span::Span; use rustc_target::abi::{HasDataLayout, TargetDataLayout}; -use crate::errors::{Abi, Align, HomogeneousAggregate, LayoutOf, Size, UnrecognizedField}; +use crate::errors::{ + LayoutAbi, LayoutAlign, LayoutHomogeneousAggregate, LayoutInvalidAttribute, LayoutOf, + LayoutSize, UnrecognizedField, +}; pub fn test_layout(tcx: TyCtxt<'_>) { if !tcx.features().rustc_attrs { @@ -16,12 +19,22 @@ pub fn test_layout(tcx: TyCtxt<'_>) { return; } for id in tcx.hir().items() { - if matches!( - tcx.def_kind(id.owner_id), - DefKind::TyAlias { .. } | DefKind::Enum | DefKind::Struct | DefKind::Union - ) { - for attr in tcx.get_attrs(id.owner_id, sym::rustc_layout) { - dump_layout_of(tcx, id.owner_id.def_id, attr); + for attr in tcx.get_attrs(id.owner_id, sym::rustc_layout) { + match tcx.def_kind(id.owner_id) { + DefKind::TyAlias { .. } | DefKind::Enum | DefKind::Struct | DefKind::Union => { + dump_layout_of(tcx, id.owner_id.def_id, attr); + } + _ => { + tcx.sess.emit_err(LayoutInvalidAttribute { span: tcx.def_span(id.owner_id) }); + } + } + } + if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. }) { + // To find associated functions we need to go into the child items here. + for &id in tcx.associated_item_def_ids(id.owner_id) { + for _attr in tcx.get_attrs(id, sym::rustc_layout) { + tcx.sess.emit_err(LayoutInvalidAttribute { span: tcx.def_span(id) }); + } } } } @@ -38,28 +51,28 @@ fn dump_layout_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId, attr: &Attribute) { for meta_item in meta_items { match meta_item.name_or_empty() { sym::abi => { - tcx.sess.emit_err(Abi { + tcx.sess.emit_err(LayoutAbi { span: tcx.def_span(item_def_id.to_def_id()), abi: format!("{:?}", ty_layout.abi), }); } sym::align => { - tcx.sess.emit_err(Align { + tcx.sess.emit_err(LayoutAlign { span: tcx.def_span(item_def_id.to_def_id()), align: format!("{:?}", ty_layout.align), }); } sym::size => { - tcx.sess.emit_err(Size { + tcx.sess.emit_err(LayoutSize { span: tcx.def_span(item_def_id.to_def_id()), size: format!("{:?}", ty_layout.size), }); } sym::homogeneous_aggregate => { - tcx.sess.emit_err(HomogeneousAggregate { + tcx.sess.emit_err(LayoutHomogeneousAggregate { span: tcx.def_span(item_def_id.to_def_id()), homogeneous_aggregate: format!( "{:?}", diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs index 906a36cdb25..aedc7b22725 100644 --- a/compiler/rustc_privacy/src/lib.rs +++ b/compiler/rustc_privacy/src/lib.rs @@ -1463,14 +1463,15 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> { }; let vis = self.tcx.local_visibility(local_def_id); - let hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id); let span = self.tcx.def_span(self.item_def_id.to_def_id()); let vis_span = self.tcx.def_span(def_id); if self.in_assoc_ty && !vis.is_at_least(self.required_visibility, self.tcx) { let vis_descr = match vis { ty::Visibility::Public => "public", ty::Visibility::Restricted(vis_def_id) => { - if vis_def_id == self.tcx.parent_module(hir_id).to_local_def_id() { + if vis_def_id + == self.tcx.parent_module_from_def_id(local_def_id).to_local_def_id() + { "private" } else if vis_def_id.is_top_level_module() { "crate-private" @@ -1504,7 +1505,7 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> { }; self.tcx.emit_spanned_lint( lint, - hir_id, + self.tcx.hir().local_def_id_to_hir_id(self.item_def_id), span, PrivateInterfacesOrBoundsLint { item_span: span, diff --git a/compiler/rustc_query_system/src/dep_graph/edges.rs b/compiler/rustc_query_system/src/dep_graph/edges.rs new file mode 100644 index 00000000000..6ba3924f65e --- /dev/null +++ b/compiler/rustc_query_system/src/dep_graph/edges.rs @@ -0,0 +1,73 @@ +use crate::dep_graph::DepNodeIndex; +use smallvec::SmallVec; +use std::hash::{Hash, Hasher}; +use std::iter::Extend; +use std::ops::Deref; + +#[derive(Default, Debug)] +pub struct EdgesVec { + max: u32, + edges: SmallVec<[DepNodeIndex; EdgesVec::INLINE_CAPACITY]>, +} + +impl Hash for EdgesVec { + #[inline] + fn hash<H: Hasher>(&self, hasher: &mut H) { + Hash::hash(&self.edges, hasher) + } +} + +impl EdgesVec { + pub const INLINE_CAPACITY: usize = 8; + + #[inline] + pub fn new() -> Self { + Self::default() + } + + #[inline] + pub fn push(&mut self, edge: DepNodeIndex) { + self.max = self.max.max(edge.as_u32()); + self.edges.push(edge); + } + + #[inline] + pub fn max_index(&self) -> u32 { + self.max + } +} + +impl Deref for EdgesVec { + type Target = [DepNodeIndex]; + + #[inline] + fn deref(&self) -> &Self::Target { + self.edges.as_slice() + } +} + +impl FromIterator<DepNodeIndex> for EdgesVec { + #[inline] + fn from_iter<T>(iter: T) -> Self + where + T: IntoIterator<Item = DepNodeIndex>, + { + let mut vec = EdgesVec::new(); + for index in iter { + vec.push(index) + } + vec + } +} + +impl Extend<DepNodeIndex> for EdgesVec { + #[inline] + fn extend<T>(&mut self, iter: T) + where + T: IntoIterator<Item = DepNodeIndex>, + { + for elem in iter { + self.push(elem); + } + } +} diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 0d4d13ac20d..7b7981e1425 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -8,7 +8,6 @@ use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering}; use rustc_data_structures::unord::UnordMap; use rustc_index::IndexVec; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder}; -use smallvec::{smallvec, SmallVec}; use std::assert_matches::assert_matches; use std::collections::hash_map::Entry; use std::fmt::Debug; @@ -19,6 +18,7 @@ use std::sync::atomic::Ordering::Relaxed; use super::query::DepGraphQuery; use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex}; use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId}; +use crate::dep_graph::EdgesVec; use crate::ich::StableHashingContext; use crate::query::{QueryContext, QuerySideEffects}; @@ -137,7 +137,7 @@ impl<K: DepKind> DepGraph<K> { let _green_node_index = current.intern_new_node( profiler, DepNode { kind: DepKind::NULL, hash: current.anon_id_seed.into() }, - smallvec![], + EdgesVec::new(), Fingerprint::ZERO, ); assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE); @@ -147,7 +147,7 @@ impl<K: DepKind> DepGraph<K> { profiler, &prev_graph, DepNode { kind: DepKind::RED, hash: Fingerprint::ZERO.into() }, - smallvec![], + EdgesVec::new(), None, false, ); @@ -356,12 +356,12 @@ impl<K: DepKind> DepGraphData<K> { let with_deps = |task_deps| K::with_deps(task_deps, || task(cx, arg)); let (result, edges) = if cx.dep_context().is_eval_always(key.kind) { - (with_deps(TaskDepsRef::EvalAlways), smallvec![]) + (with_deps(TaskDepsRef::EvalAlways), EdgesVec::new()) } else { let task_deps = Lock::new(TaskDeps { #[cfg(debug_assertions)] node: Some(key), - reads: SmallVec::new(), + reads: EdgesVec::new(), read_set: Default::default(), phantom_data: PhantomData, }); @@ -486,14 +486,14 @@ impl<K: DepKind> DepGraph<K> { // As long as we only have a low number of reads we can avoid doing a hash // insert and potentially allocating/reallocating the hashmap - let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP { + let new_read = if task_deps.reads.len() < EdgesVec::INLINE_CAPACITY { task_deps.reads.iter().all(|other| *other != dep_node_index) } else { task_deps.read_set.insert(dep_node_index) }; if new_read { task_deps.reads.push(dep_node_index); - if task_deps.reads.len() == TASK_DEPS_READS_CAP { + if task_deps.reads.len() == EdgesVec::INLINE_CAPACITY { // Fill `read_set` with what we have so far so we can use the hashset // next time task_deps.read_set.extend(task_deps.reads.iter().copied()); @@ -572,7 +572,7 @@ impl<K: DepKind> DepGraph<K> { } } - let mut edges = SmallVec::new(); + let mut edges = EdgesVec::new(); K::read_deps(|task_deps| match task_deps { TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()), TaskDepsRef::EvalAlways => { @@ -872,7 +872,7 @@ impl<K: DepKind> DepGraphData<K> { let prev_deps = self.previous.edge_targets_from(prev_dep_node_index); - for &dep_dep_node_index in prev_deps { + for dep_dep_node_index in prev_deps { self.try_mark_parent_green(qcx, dep_dep_node_index, dep_node, Some(&frame))?; } @@ -1308,8 +1308,7 @@ impl<K: DepKind> CurrentDepGraph<K> { let key = prev_graph.index_to_node(prev_index); let edges = prev_graph .edge_targets_from(prev_index) - .iter() - .map(|i| prev_index_to_index[*i].unwrap()) + .map(|i| prev_index_to_index[i].unwrap()) .collect(); let fingerprint = prev_graph.fingerprint_by_index(prev_index); let dep_node_index = self.encoder.borrow().send(profiler, key, fingerprint, edges); @@ -1335,10 +1334,6 @@ impl<K: DepKind> CurrentDepGraph<K> { } } -/// The capacity of the `reads` field `SmallVec` -const TASK_DEPS_READS_CAP: usize = 8; -type EdgesVec = SmallVec<[DepNodeIndex; TASK_DEPS_READS_CAP]>; - #[derive(Debug, Clone, Copy)] pub enum TaskDepsRef<'a, K: DepKind> { /// New dependencies can be added to the diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index 0fd9e35d6dc..272028d35bf 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -1,10 +1,12 @@ pub mod debug; mod dep_node; +mod edges; mod graph; mod query; mod serialized; pub use dep_node::{DepKindStruct, DepNode, DepNodeParams, WorkProductId}; +pub use edges::EdgesVec; pub use graph::{ hash_result, DepGraph, DepGraphData, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef, WorkProduct, WorkProductMap, @@ -157,4 +159,10 @@ pub trait DepKind: Copy + fmt::Debug + Eq + Hash + Send + Encodable<FileEncoder> fn read_deps<OP>(op: OP) where OP: for<'a> FnOnce(TaskDepsRef<'a, Self>); + + fn from_u16(u: u16) -> Self; + + fn to_u16(self) -> u16; + + const MAX: u16; } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index edddfda6242..4ba0cb31d0b 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -1,6 +1,6 @@ //! The data that we will serialize and deserialize. //! -//! The dep-graph is serialized as a sequence of NodeInfo, with the dependencies +//! Notionally, the dep-graph is a sequence of NodeInfo with the dependencies //! specified inline. The total number of nodes and edges are stored as the last //! 16 bytes of the file, so we can find them easily at decoding time. //! @@ -11,17 +11,42 @@ //! sequence of NodeInfos to the different arrays in SerializedDepGraph. Since the //! node and edge count are stored at the end of the file, all the arrays can be //! pre-allocated with the right length. +//! +//! The encoding of the de-pgraph is generally designed around the fact that fixed-size +//! reads of encoded data are generally faster than variable-sized reads. Ergo we adopt +//! essentially the same varint encoding scheme used in the rmeta format; the edge lists +//! for each node on the graph store a 2-bit integer which is the number of bytes per edge +//! index in that node's edge list. We effectively ignore that an edge index of 0 could be +//! encoded with 0 bytes in order to not require 3 bits to store the byte width of the edges. +//! The overhead of calculating the correct byte width for each edge is mitigated by +//! building edge lists with [`EdgesVec`] which keeps a running max of the edges in a node. +//! +//! When we decode this data, we do not immediately create [`SerializedDepNodeIndex`] and +//! instead keep the data in its denser serialized form which lets us turn our on-disk size +//! efficiency directly into a peak memory reduction. When we convert these encoded-in-memory +//! values into their fully-deserialized type, we use a fixed-size read of the encoded array +//! then mask off any errant bytes we read. The array of edge index bytes is padded to permit this. +//! +//! We also encode and decode the entire rest of each node using [`SerializedNodeHeader`] +//! to let this encoding and decoding be done in one fixed-size operation. These headers contain +//! two [`Fingerprint`]s along with the serialized [`DepKind`], and the number of edge indices +//! in the node and the number of bytes used to encode the edge indices for this node. The +//! [`DepKind`], number of edges, and bytes per edge are all bit-packed together, if they fit. +//! If the number of edges in this node does not fit in the bits available in the header, we +//! store it directly after the header with leb128. use super::query::DepGraphQuery; use super::{DepKind, DepNode, DepNodeIndex}; +use crate::dep_graph::EdgesVec; use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::fingerprint::PackedFingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::sync::Lock; use rustc_index::{Idx, IndexVec}; use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}; -use rustc_serialize::{Decodable, Decoder, Encodable}; -use smallvec::SmallVec; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; +use std::marker::PhantomData; // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits // unused so that we can store multiple index types in `CompressedHybridIndex`, @@ -31,6 +56,16 @@ rustc_index::newtype_index! { pub struct SerializedDepNodeIndex {} } +const DEP_NODE_SIZE: usize = std::mem::size_of::<SerializedDepNodeIndex>(); +/// Amount of padding we need to add to the edge list data so that we can retrieve every +/// SerializedDepNodeIndex with a fixed-size read then mask. +const DEP_NODE_PAD: usize = DEP_NODE_SIZE - 1; +/// Number of bits we need to store the number of used bytes in a SerializedDepNodeIndex. +/// Note that wherever we encode byte widths like this we actually store the number of bytes used +/// minus 1; for a 4-byte value we technically would have 5 widths to store, but using one byte to +/// store zeroes (which are relatively rare) is a decent tradeoff to save a bit in our bitfields. +const DEP_NODE_WIDTH_BITS: usize = DEP_NODE_SIZE / 2; + /// Data for use when recompiling the **current crate**. #[derive(Debug)] pub struct SerializedDepGraph<K: DepKind> { @@ -42,10 +77,10 @@ pub struct SerializedDepGraph<K: DepKind> { /// For each DepNode, stores the list of edges originating from that /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, /// which holds the actual DepNodeIndices of the target nodes. - edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>, - /// A flattened list of all edge targets in the graph. Edge sources are - /// implicit in edge_list_indices. - edge_list_data: Vec<SerializedDepNodeIndex>, + edge_list_indices: IndexVec<SerializedDepNodeIndex, EdgeHeader>, + /// A flattened list of all edge targets in the graph, stored in the same + /// varint encoding that we use on disk. Edge sources are implicit in edge_list_indices. + edge_list_data: Vec<u8>, /// Reciprocal map to `nodes`. index: FxHashMap<DepNode<K>, SerializedDepNodeIndex>, } @@ -64,9 +99,35 @@ impl<K: DepKind> Default for SerializedDepGraph<K> { impl<K: DepKind> SerializedDepGraph<K> { #[inline] - pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] { - let targets = self.edge_list_indices[source]; - &self.edge_list_data[targets.0 as usize..targets.1 as usize] + pub fn edge_targets_from( + &self, + source: SerializedDepNodeIndex, + ) -> impl Iterator<Item = SerializedDepNodeIndex> + '_ { + let header = self.edge_list_indices[source]; + let mut raw = &self.edge_list_data[header.start()..]; + // Figure out where the edge list for `source` ends by getting the start index of the next + // edge list, or the end of the array if this is the last edge. + let end = self + .edge_list_indices + .get(source + 1) + .map(|h| h.start()) + .unwrap_or_else(|| self.edge_list_data.len() - DEP_NODE_PAD); + + // The number of edges for this node is implicitly stored in the combination of the byte + // width and the length. + let bytes_per_index = header.bytes_per_index(); + let len = (end - header.start()) / bytes_per_index; + + // LLVM doesn't hoist EdgeHeader::mask so we do it ourselves. + let mask = header.mask(); + (0..len).map(move |_| { + // Doing this slicing in this order ensures that the first bounds check suffices for + // all the others. + let index = &raw[..DEP_NODE_SIZE]; + raw = &raw[bytes_per_index..]; + let index = u32::from_le_bytes(index.try_into().unwrap()) & mask; + SerializedDepNodeIndex::from_u32(index) + }) } #[inline] @@ -84,11 +145,42 @@ impl<K: DepKind> SerializedDepGraph<K> { self.fingerprints[dep_node_index] } + #[inline] pub fn node_count(&self) -> usize { self.index.len() } } +/// A packed representation of an edge's start index and byte width. +/// +/// This is packed by stealing 2 bits from the start index, which means we only accomodate edge +/// data arrays up to a quarter of our address space. Which seems fine. +#[derive(Debug, Clone, Copy)] +struct EdgeHeader { + repr: usize, +} + +impl EdgeHeader { + #[inline] + fn start(self) -> usize { + self.repr >> DEP_NODE_WIDTH_BITS + } + + #[inline] + fn bytes_per_index(self) -> usize { + (self.repr & mask(DEP_NODE_WIDTH_BITS)) + 1 + } + + #[inline] + fn mask(self) -> u32 { + mask(self.bytes_per_index() * 8) as u32 + } +} + +fn mask(bits: usize) -> usize { + usize::MAX >> ((std::mem::size_of::<usize>() * 8) - bits) +} + impl<'a, K: DepKind + Decodable<MemDecoder<'a>>> Decodable<MemDecoder<'a>> for SerializedDepGraph<K> { @@ -107,32 +199,58 @@ impl<'a, K: DepKind + Decodable<MemDecoder<'a>>> Decodable<MemDecoder<'a>> debug!(?node_count, ?edge_count); + let graph_bytes = d.len() - (2 * IntEncodedWithFixedSize::ENCODED_SIZE) - d.position(); + let mut nodes = IndexVec::with_capacity(node_count); let mut fingerprints = IndexVec::with_capacity(node_count); let mut edge_list_indices = IndexVec::with_capacity(node_count); - let mut edge_list_data = Vec::with_capacity(edge_count); + // This estimation assumes that all of the encoded bytes are for the edge lists or for the + // fixed-size node headers. But that's not necessarily true; if any edge list has a length + // that spills out of the size we can bit-pack into SerializedNodeHeader then some of the + // total serialized size is also used by leb128-encoded edge list lengths. Neglecting that + // contribution to graph_bytes means our estimation of the bytes needed for edge_list_data + // slightly overshoots. But it cannot overshoot by much; consider that the worse case is + // for a node with length 64, which means the spilled 1-byte leb128 length is 1 byte of at + // least (34 byte header + 1 byte len + 64 bytes edge data), which is ~1%. A 2-byte leb128 + // length is about the same fractional overhead and it amortizes for yet greater lengths. + let mut edge_list_data = Vec::with_capacity( + graph_bytes - node_count * std::mem::size_of::<SerializedNodeHeader<K>>(), + ); for _index in 0..node_count { - let dep_node: DepNode<K> = Decodable::decode(d); - let _i: SerializedDepNodeIndex = nodes.push(dep_node); + // Decode the header for this edge; the header packs together as many of the fixed-size + // fields as possible to limit the number of times we update decoder state. + let node_header = SerializedNodeHeader { bytes: d.read_array(), _marker: PhantomData }; + + let _i: SerializedDepNodeIndex = nodes.push(node_header.node()); debug_assert_eq!(_i.index(), _index); - let fingerprint: Fingerprint = Decodable::decode(d); - let _i: SerializedDepNodeIndex = fingerprints.push(fingerprint); + let _i: SerializedDepNodeIndex = fingerprints.push(node_header.fingerprint()); debug_assert_eq!(_i.index(), _index); - // Deserialize edges -- sequence of DepNodeIndex - let len = d.read_usize(); - let start = edge_list_data.len().try_into().unwrap(); - for _ in 0..len { - let edge = Decodable::decode(d); - edge_list_data.push(edge); - } - let end = edge_list_data.len().try_into().unwrap(); - let _i: SerializedDepNodeIndex = edge_list_indices.push((start, end)); + // If the length of this node's edge list is small, the length is stored in the header. + // If it is not, we fall back to another decoder call. + let num_edges = node_header.len().unwrap_or_else(|| d.read_usize()); + + // The edges index list uses the same varint strategy as rmeta tables; we select the + // number of byte elements per-array not per-element. This lets us read the whole edge + // list for a node with one decoder call and also use the on-disk format in memory. + let edges_len_bytes = node_header.bytes_per_index() * num_edges; + // The in-memory structure for the edges list stores the byte width of the edges on + // this node with the offset into the global edge data array. + let edges_header = node_header.edges_header(&edge_list_data); + + edge_list_data.extend(d.read_raw_bytes(edges_len_bytes)); + + let _i: SerializedDepNodeIndex = edge_list_indices.push(edges_header); debug_assert_eq!(_i.index(), _index); } + // When we access the edge list data, we do a fixed-size read from the edge list data then + // mask off the bytes that aren't for that edge index, so the last read may dangle off the + // end of the array. This padding ensure it doesn't. + edge_list_data.extend(&[0u8; DEP_NODE_PAD]); + let index: FxHashMap<_, _> = nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect(); @@ -140,11 +258,154 @@ impl<'a, K: DepKind + Decodable<MemDecoder<'a>>> Decodable<MemDecoder<'a>> } } -#[derive(Debug, Encodable, Decodable)] -pub struct NodeInfo<K: DepKind> { +/// A packed representation of all the fixed-size fields in a `NodeInfo`. +/// +/// This stores in one byte array: +/// * The `Fingerprint` in the `NodeInfo` +/// * The `Fingerprint` in `DepNode` that is in this `NodeInfo` +/// * The `DepKind`'s discriminant (a u16, but not all bits are used...) +/// * The byte width of the encoded edges for this node +/// * In whatever bits remain, the length of the edge list for this node, if it fits +struct SerializedNodeHeader<K> { + // 2 bytes for the DepNode + // 16 for Fingerprint in DepNode + // 16 for Fingerprint in NodeInfo + bytes: [u8; 34], + _marker: PhantomData<K>, +} + +// The fields of a `SerializedNodeHeader`, this struct is an implementation detail and exists only +// to make the implementation of `SerializedNodeHeader` simpler. +struct Unpacked<K> { + len: Option<usize>, + bytes_per_index: usize, + kind: K, + hash: PackedFingerprint, + fingerprint: Fingerprint, +} + +// Bit fields, where +// M: bits used to store the length of a node's edge list +// N: bits used to store the byte width of elements of the edge list +// are +// 0..M length of the edge +// M..M+N bytes per index +// M+N..16 kind +impl<K: DepKind> SerializedNodeHeader<K> { + const TOTAL_BITS: usize = std::mem::size_of::<K>() * 8; + const LEN_BITS: usize = Self::TOTAL_BITS - Self::KIND_BITS - Self::WIDTH_BITS; + const WIDTH_BITS: usize = DEP_NODE_WIDTH_BITS; + const KIND_BITS: usize = Self::TOTAL_BITS - K::MAX.leading_zeros() as usize; + const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1; + + #[inline] + fn new(node_info: &NodeInfo<K>) -> Self { + debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS); + + let NodeInfo { node, fingerprint, edges } = node_info; + + let mut head = node.kind.to_u16(); + + let free_bytes = edges.max_index().leading_zeros() as usize / 8; + let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1); + head |= (bytes_per_index as u16) << Self::KIND_BITS; + + // Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit + // in this bitfield. + if edges.len() <= Self::MAX_INLINE_LEN { + head |= (edges.len() as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS); + } + + let hash: Fingerprint = node.hash.into(); + + // Using half-open ranges ensures an unconditional panic if we get the magic numbers wrong. + let mut bytes = [0u8; 34]; + bytes[..2].copy_from_slice(&head.to_le_bytes()); + bytes[2..18].copy_from_slice(&hash.to_le_bytes()); + bytes[18..].copy_from_slice(&fingerprint.to_le_bytes()); + + #[cfg(debug_assertions)] + { + let res = Self { bytes, _marker: PhantomData }; + assert_eq!(node_info.fingerprint, res.fingerprint()); + assert_eq!(node_info.node, res.node()); + if let Some(len) = res.len() { + assert_eq!(node_info.edges.len(), len); + } + } + Self { bytes, _marker: PhantomData } + } + + #[inline] + fn unpack(&self) -> Unpacked<K> { + let head = u16::from_le_bytes(self.bytes[..2].try_into().unwrap()); + let hash = self.bytes[2..18].try_into().unwrap(); + let fingerprint = self.bytes[18..].try_into().unwrap(); + + let kind = head & mask(Self::KIND_BITS) as u16; + let bytes_per_index = (head >> Self::KIND_BITS) & mask(Self::WIDTH_BITS) as u16; + let len = (head as usize) >> (Self::WIDTH_BITS + Self::KIND_BITS); + + Unpacked { + len: len.checked_sub(1), + bytes_per_index: bytes_per_index as usize + 1, + kind: DepKind::from_u16(kind), + hash: Fingerprint::from_le_bytes(hash).into(), + fingerprint: Fingerprint::from_le_bytes(fingerprint), + } + } + + #[inline] + fn len(&self) -> Option<usize> { + self.unpack().len + } + + #[inline] + fn bytes_per_index(&self) -> usize { + self.unpack().bytes_per_index + } + + #[inline] + fn fingerprint(&self) -> Fingerprint { + self.unpack().fingerprint + } + + #[inline] + fn node(&self) -> DepNode<K> { + let Unpacked { kind, hash, .. } = self.unpack(); + DepNode { kind, hash } + } + + #[inline] + fn edges_header(&self, edge_list_data: &[u8]) -> EdgeHeader { + EdgeHeader { + repr: (edge_list_data.len() << DEP_NODE_WIDTH_BITS) | (self.bytes_per_index() - 1), + } + } +} + +#[derive(Debug)] +struct NodeInfo<K: DepKind> { node: DepNode<K>, fingerprint: Fingerprint, - edges: SmallVec<[DepNodeIndex; 8]>, + edges: EdgesVec, +} + +impl<K: DepKind> Encodable<FileEncoder> for NodeInfo<K> { + fn encode(&self, e: &mut FileEncoder) { + let header = SerializedNodeHeader::new(self); + e.emit_raw_bytes(&header.bytes); + + if header.len().is_none() { + e.emit_usize(self.edges.len()); + } + + let bytes_per_index = header.bytes_per_index(); + for node_index in self.edges.iter() { + let bytes = node_index.as_u32().to_le_bytes(); + e.emit_raw_bytes(&bytes[..bytes_per_index]); + } + } } struct Stat<K: DepKind> { @@ -303,7 +564,7 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> { profiler: &SelfProfilerRef, node: DepNode<K>, fingerprint: Fingerprint, - edges: SmallVec<[DepNodeIndex; 8]>, + edges: EdgesVec, ) -> DepNodeIndex { let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); let node = NodeInfo { node, fingerprint, edges }; diff --git a/compiler/rustc_query_system/src/ich/impls_syntax.rs b/compiler/rustc_query_system/src/ich/impls_syntax.rs index f2b13f95def..b2177be0e36 100644 --- a/compiler/rustc_query_system/src/ich/impls_syntax.rs +++ b/compiler/rustc_query_system/src/ich/impls_syntax.rs @@ -79,15 +79,16 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile { src_hash.hash_stable(hcx, hasher); - // We are always in `Lines` form by the time we reach here. - assert!(self.lines.borrow().is_lines()); - self.lines(|lines| { + { + // We are always in `Lines` form by the time we reach here. + assert!(self.lines.read().is_lines()); + let lines = self.lines(); // We only hash the relative position within this source_file lines.len().hash_stable(hcx, hasher); for &line in lines.iter() { line.hash_stable(hcx, hasher); } - }); + } // We only hash the relative position within this source_file multibyte_chars.len().hash_stable(hcx, hasher); diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs index 8c9e9cfad60..1944ac443ea 100644 --- a/compiler/rustc_query_system/src/lib.rs +++ b/compiler/rustc_query_system/src/lib.rs @@ -4,6 +4,7 @@ #![feature(min_specialization)] #![feature(extern_types)] #![feature(let_chains)] +#![feature(inline_const)] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs index d99fc07a7cd..e9edfb53b44 100644 --- a/compiler/rustc_resolve/src/diagnostics.rs +++ b/compiler/rustc_resolve/src/diagnostics.rs @@ -2753,7 +2753,13 @@ fn search_for_any_use_in_items(items: &[P<ast::Item>]) -> Option<Span> { for item in items { if let ItemKind::Use(..) = item.kind { if is_span_suitable_for_use_injection(item.span) { - return Some(item.span.shrink_to_lo()); + let mut lo = item.span.lo(); + for attr in &item.attrs { + if attr.span.eq_ctxt(item.span) { + lo = std::cmp::min(lo, attr.span.lo()); + } + } + return Some(Span::new(lo, lo, item.span.ctxt(), item.span.parent())); } } } diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs index 0ffc537eee0..f1b7e8d9ae0 100644 --- a/compiler/rustc_serialize/src/opaque.rs +++ b/compiler/rustc_serialize/src/opaque.rs @@ -353,7 +353,7 @@ impl<'a> MemDecoder<'a> { } #[inline] - fn read_array<const N: usize>(&mut self) -> [u8; N] { + pub fn read_array<const N: usize>(&mut self) -> [u8; N] { self.read_raw_bytes(N).try_into().unwrap() } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index f00472f181d..86a03124fe1 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -381,6 +381,24 @@ pub enum DebugInfo { Full, } +#[derive(Clone, Copy, Debug, PartialEq, Hash)] +pub enum DebugInfoCompression { + None, + Zlib, + Zstd, +} + +impl ToString for DebugInfoCompression { + fn to_string(&self) -> String { + match self { + DebugInfoCompression::None => "none", + DebugInfoCompression::Zlib => "zlib", + DebugInfoCompression::Zstd => "zstd", + } + .to_owned() + } +} + /// Split debug-information is enabled by `-C split-debuginfo`, this enum is only used if split /// debug-information is enabled (in either `Packed` or `Unpacked` modes), and the platform /// uses DWARF for debug-information. @@ -1015,6 +1033,7 @@ impl Default for Options { crate_types: Vec::new(), optimize: OptLevel::No, debuginfo: DebugInfo::None, + debuginfo_compression: DebugInfoCompression::None, lint_opts: Vec::new(), lint_cap: None, describe_lints: false, @@ -2160,12 +2179,6 @@ fn collect_print_requests( prints.extend(matches.opt_strs("print").into_iter().map(|req| { let (req, out) = split_out_file_name(&req); - if out.is_some() && !unstable_opts.unstable_options { - handler.early_error( - "the `-Z unstable-options` flag must also be passed to \ - enable the path print option", - ); - } let kind = match PRINT_KINDS.iter().find(|&&(name, _)| name == req) { Some((_, PrintKind::TargetSpec)) => { if unstable_opts.unstable_options { @@ -2283,6 +2296,13 @@ fn select_debuginfo(matches: &getopts::Matches, cg: &CodegenOptions) -> DebugInf if max_g > max_c { DebugInfo::Full } else { cg.debuginfo } } +fn select_debuginfo_compression( + _handler: &EarlyErrorHandler, + unstable_opts: &UnstableOptions, +) -> DebugInfoCompression { + unstable_opts.debuginfo_compression +} + pub(crate) fn parse_assert_incr_state( handler: &EarlyErrorHandler, opt_assertion: &Option<String>, @@ -2758,6 +2778,8 @@ pub fn build_session_options( // for more details. let debug_assertions = cg.debug_assertions.unwrap_or(opt_level == OptLevel::No); let debuginfo = select_debuginfo(matches, &cg); + let debuginfo_compression: DebugInfoCompression = + select_debuginfo_compression(handler, &unstable_opts); let mut search_paths = vec![]; for s in &matches.opt_strs("L") { @@ -2834,6 +2856,7 @@ pub fn build_session_options( crate_types, optimize: opt_level, debuginfo, + debuginfo_compression, lint_opts, lint_cap, describe_lints, @@ -3119,11 +3142,11 @@ impl PpMode { /// how the hash should be calculated when adding a new command-line argument. pub(crate) mod dep_tracking { use super::{ - BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, ErrorOutputType, - InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, LocationDetail, LtoCli, - OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Passes, ResolveDocLinks, - SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, - TraitSolver, TrimmedDefPaths, + BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression, + ErrorOutputType, InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, + LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, + Passes, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, + SymbolManglingVersion, TraitSolver, TrimmedDefPaths, }; use crate::lint; use crate::options::WasiExecModel; @@ -3201,6 +3224,7 @@ pub(crate) mod dep_tracking { OptLevel, LtoCli, DebugInfo, + DebugInfoCompression, UnstableFeatures, NativeLib, NativeLibKind, diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs index c53a355b533..90f57be9324 100644 --- a/compiler/rustc_session/src/cstore.rs +++ b/compiler/rustc_session/src/cstore.rs @@ -7,7 +7,7 @@ use crate::utils::NativeLibKind; use crate::Session; use rustc_ast as ast; use rustc_data_structures::owned_slice::OwnedSlice; -use rustc_data_structures::sync::{self, AppendOnlyIndexVec, RwLock}; +use rustc_data_structures::sync::{self, AppendOnlyIndexVec, FreezeLock, RwLock}; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, StableCrateId, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash, Definitions}; use rustc_span::hygiene::{ExpnHash, ExpnId}; @@ -261,5 +261,5 @@ pub struct Untracked { pub cstore: RwLock<Box<CrateStoreDyn>>, /// Reference span for definitions. pub source_span: AppendOnlyIndexVec<LocalDefId, Span>, - pub definitions: RwLock<Definitions>, + pub definitions: FreezeLock<Definitions>, } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 40099de707b..6c26859228c 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -139,6 +139,7 @@ top_level_options!( /// can influence whether overflow checks are done or not. debug_assertions: bool [TRACKED], debuginfo: DebugInfo [TRACKED], + debuginfo_compression: DebugInfoCompression [TRACKED], lint_opts: Vec<(String, lint::Level)> [TRACKED_NO_CRATE_HASH], lint_cap: Option<lint::Level> [TRACKED_NO_CRATE_HASH], describe_lints: bool [UNTRACKED], @@ -376,6 +377,7 @@ mod desc { "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`"; pub const parse_cfprotection: &str = "`none`|`no`|`n` (default), `branch`, `return`, or `full`|`yes`|`y` (equivalent to `branch` and `return`)"; pub const parse_debuginfo: &str = "either an integer (0, 1, 2), `none`, `line-directives-only`, `line-tables-only`, `limited`, or `full`"; + pub const parse_debuginfo_compression: &str = "one of `none`, `zlib`, or `zstd`"; pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`"; pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of(); pub const parse_optimization_fuel: &str = "crate=integer"; @@ -782,6 +784,19 @@ mod parse { true } + pub(crate) fn parse_debuginfo_compression( + slot: &mut DebugInfoCompression, + v: Option<&str>, + ) -> bool { + match v { + Some("none") => *slot = DebugInfoCompression::None, + Some("zlib") => *slot = DebugInfoCompression::Zlib, + Some("zstd") => *slot = DebugInfoCompression::Zstd, + _ => return false, + }; + true + } + pub(crate) fn parse_linker_flavor(slot: &mut Option<LinkerFlavorCli>, v: Option<&str>) -> bool { match v.and_then(LinkerFlavorCli::from_str) { Some(lf) => *slot = Some(lf), @@ -1424,6 +1439,8 @@ options! { "emit discriminators and other data necessary for AutoFDO"), debug_macros: bool = (false, parse_bool, [TRACKED], "emit line numbers debug info inside macros (default: no)"), + debuginfo_compression: DebugInfoCompression = (DebugInfoCompression::None, parse_debuginfo_compression, [TRACKED], + "compress debug info sections (none, zlib, zstd, default: none)"), deduplicate_diagnostics: bool = (true, parse_bool, [UNTRACKED], "deduplicate identical diagnostics (default: yes)"), dep_info_omit_d_target: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 86db2edab7b..9bff9017881 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -204,6 +204,12 @@ pub struct Session { /// The version of the rustc process, possibly including a commit hash and description. pub cfg_version: &'static str, + + /// All commandline args used to invoke the compiler, with @file args fully expanded. + /// This will only be used within debug info, e.g. in the pdb file on windows + /// This is mainly useful for other tools that reads that debuginfo to figure out + /// how to call the compiler with the same arguments. + pub expanded_args: Vec<String>, } pub struct PerfStats { @@ -1325,6 +1331,7 @@ pub fn build_session( target_override: Option<Target>, cfg_version: &'static str, ice_file: Option<PathBuf>, + expanded_args: Vec<String>, ) -> Session { // FIXME: This is not general enough to make the warning lint completely override // normal diagnostic warnings, since the warning lint can also be denied and changed @@ -1467,6 +1474,7 @@ pub fn build_session( target_features: Default::default(), unstable_target_features: Default::default(), cfg_version, + expanded_args, }; validate_commandline_args_with_session_available(&sess); diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs index 52ba4bd4e57..a7640736481 100644 --- a/compiler/rustc_smir/src/rustc_smir/mod.rs +++ b/compiler/rustc_smir/src/rustc_smir/mod.rs @@ -78,8 +78,8 @@ impl<'tcx> Context for Tables<'tcx> { impl_trait.stable(self) } - fn mir_body(&mut self, item: &stable_mir::CrateItem) -> stable_mir::mir::Body { - let def_id = self[item.0]; + fn mir_body(&mut self, item: stable_mir::DefId) -> stable_mir::mir::Body { + let def_id = self[item]; let mir = self.tcx.optimized_mir(def_id); stable_mir::mir::Body { blocks: mir @@ -103,8 +103,13 @@ impl<'tcx> Context for Tables<'tcx> { } fn ty_kind(&mut self, ty: crate::stable_mir::ty::Ty) -> TyKind { - let ty = self.types[ty.0]; - ty.stable(self) + self.types[ty.0].clone().stable(self) + } + + fn mk_ty(&mut self, kind: TyKind) -> stable_mir::ty::Ty { + let n = self.types.len(); + self.types.push(MaybeStable::Stable(kind)); + stable_mir::ty::Ty(n) } fn generics_of(&mut self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics { @@ -128,20 +133,47 @@ impl<'tcx> Context for Tables<'tcx> { } } +#[derive(Clone)] +pub enum MaybeStable<S, R> { + Stable(S), + Rustc(R), +} + +impl<'tcx, S, R> MaybeStable<S, R> { + fn stable(self, tables: &mut Tables<'tcx>) -> S + where + R: Stable<'tcx, T = S>, + { + match self { + MaybeStable::Stable(s) => s, + MaybeStable::Rustc(r) => r.stable(tables), + } + } +} + +impl<S, R: PartialEq> PartialEq<R> for MaybeStable<S, R> { + fn eq(&self, other: &R) -> bool { + match self { + MaybeStable::Stable(_) => false, + MaybeStable::Rustc(r) => r == other, + } + } +} + pub struct Tables<'tcx> { pub tcx: TyCtxt<'tcx>, pub def_ids: Vec<DefId>, pub alloc_ids: Vec<AllocId>, - pub types: Vec<Ty<'tcx>>, + pub types: Vec<MaybeStable<stable_mir::ty::TyKind, Ty<'tcx>>>, } impl<'tcx> Tables<'tcx> { fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty { - if let Some(id) = self.types.iter().position(|&t| t == ty) { + if let Some(id) = self.types.iter().position(|t| *t == ty) { return stable_mir::ty::Ty(id); } let id = self.types.len(); - self.types.push(ty); + self.types.push(MaybeStable::Rustc(ty)); stable_mir::ty::Ty(id) } } @@ -1097,14 +1129,13 @@ impl<'tcx> Stable<'tcx> for ty::Const<'tcx> { tables, )) } - ty::ParamCt(param) => stable_mir::ty::ConstantKind::ParamCt(opaque(¶m)), + ty::ParamCt(param) => stable_mir::ty::ConstantKind::Param(param.stable(tables)), ty::ErrorCt(_) => unreachable!(), ty::InferCt(_) => unreachable!(), ty::BoundCt(_, _) => unimplemented!(), ty::PlaceholderCt(_) => unimplemented!(), ty::Unevaluated(uv) => { stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst { - ty: tables.intern_ty(self.ty()), def: tables.const_def(uv.def), args: uv.args.stable(tables), promoted: None, @@ -1112,10 +1143,19 @@ impl<'tcx> Stable<'tcx> for ty::Const<'tcx> { } ty::ExprCt(_) => unimplemented!(), }, + ty: tables.intern_ty(self.ty()), } } } +impl<'tcx> Stable<'tcx> for ty::ParamConst { + type T = stable_mir::ty::ParamConst; + fn stable(&self, _: &mut Tables<'tcx>) -> Self::T { + use stable_mir::ty::ParamConst; + ParamConst { index: self.index, name: self.name.to_string() } + } +} + impl<'tcx> Stable<'tcx> for ty::ParamTy { type T = stable_mir::ty::ParamTy; fn stable(&self, _: &mut Tables<'tcx>) -> Self::T { @@ -1184,22 +1224,27 @@ impl<'tcx> Stable<'tcx> for ty::TraitDef { } impl<'tcx> Stable<'tcx> for rustc_middle::mir::ConstantKind<'tcx> { - type T = stable_mir::ty::ConstantKind; + type T = stable_mir::ty::Const; fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T { match *self { - ConstantKind::Ty(c) => c.stable(tables).literal, - ConstantKind::Unevaluated(unev_const, ty) => { - stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst { - ty: tables.intern_ty(ty), - def: tables.const_def(unev_const.def), - args: unev_const.args.stable(tables), - promoted: unev_const.promoted.map(|u| u.as_u32()), - }) - } - ConstantKind::Val(val, ty) => { - stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation(ty, val, tables)) - } + ConstantKind::Ty(c) => c.stable(tables), + ConstantKind::Unevaluated(unev_const, ty) => stable_mir::ty::Const { + literal: stable_mir::ty::ConstantKind::Unevaluated( + stable_mir::ty::UnevaluatedConst { + def: tables.const_def(unev_const.def), + args: unev_const.args.stable(tables), + promoted: unev_const.promoted.map(|u| u.as_u32()), + }, + ), + ty: tables.intern_ty(ty), + }, + ConstantKind::Val(val, ty) => stable_mir::ty::Const { + literal: stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation( + ty, val, tables, + )), + ty: tables.intern_ty(ty), + }, } } } diff --git a/compiler/rustc_smir/src/stable_mir/fold.rs b/compiler/rustc_smir/src/stable_mir/fold.rs new file mode 100644 index 00000000000..831cfb40a15 --- /dev/null +++ b/compiler/rustc_smir/src/stable_mir/fold.rs @@ -0,0 +1,230 @@ +use std::ops::ControlFlow; + +use crate::rustc_internal::Opaque; + +use super::ty::{ + Allocation, Binder, Const, ConstDef, ConstantKind, ExistentialPredicate, FnSig, GenericArgKind, + GenericArgs, Promoted, RigidTy, TermKind, Ty, TyKind, UnevaluatedConst, +}; + +pub trait Folder: Sized { + type Break; + fn visit_ty(&mut self, ty: &Ty) -> ControlFlow<Self::Break, Ty> { + ty.super_fold(self) + } + fn fold_const(&mut self, c: &Const) -> ControlFlow<Self::Break, Const> { + c.super_fold(self) + } +} + +pub trait Foldable: Sized + Clone { + fn fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + self.super_fold(folder) + } + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self>; +} + +impl Foldable for Ty { + fn fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + folder.visit_ty(self) + } + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut kind = self.kind(); + match &mut kind { + super::ty::TyKind::RigidTy(ty) => *ty = ty.fold(folder)?, + super::ty::TyKind::Alias(_, alias) => alias.args = alias.args.fold(folder)?, + super::ty::TyKind::Param(_) => {} + super::ty::TyKind::Bound(_, _) => {} + } + ControlFlow::Continue(kind.into()) + } +} + +impl Foldable for Const { + fn fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + folder.fold_const(self) + } + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut this = self.clone(); + match &mut this.literal { + super::ty::ConstantKind::Allocated(alloc) => *alloc = alloc.fold(folder)?, + super::ty::ConstantKind::Unevaluated(uv) => *uv = uv.fold(folder)?, + super::ty::ConstantKind::Param(_) => {} + } + this.ty = this.ty.fold(folder)?; + ControlFlow::Continue(this) + } +} + +impl Foldable for Opaque { + fn super_fold<V: Folder>(&self, _folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(self.clone()) + } +} + +impl Foldable for Allocation { + fn super_fold<V: Folder>(&self, _folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(self.clone()) + } +} + +impl Foldable for UnevaluatedConst { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let UnevaluatedConst { def, args, promoted } = self; + ControlFlow::Continue(UnevaluatedConst { + def: def.fold(folder)?, + args: args.fold(folder)?, + promoted: promoted.fold(folder)?, + }) + } +} + +impl Foldable for ConstDef { + fn super_fold<V: Folder>(&self, _folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(self.clone()) + } +} + +impl<T: Foldable> Foldable for Option<T> { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(match self { + Some(val) => Some(val.fold(folder)?), + None => None, + }) + } +} + +impl Foldable for Promoted { + fn super_fold<V: Folder>(&self, _folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(self.clone()) + } +} + +impl Foldable for GenericArgs { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(GenericArgs(self.0.fold(folder)?)) + } +} + +impl Foldable for GenericArgKind { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut this = self.clone(); + match &mut this { + GenericArgKind::Lifetime(lt) => *lt = lt.fold(folder)?, + GenericArgKind::Type(t) => *t = t.fold(folder)?, + GenericArgKind::Const(c) => *c = c.fold(folder)?, + } + ControlFlow::Continue(this) + } +} + +impl Foldable for RigidTy { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut this = self.clone(); + match &mut this { + RigidTy::Bool + | RigidTy::Char + | RigidTy::Int(_) + | RigidTy::Uint(_) + | RigidTy::Float(_) + | RigidTy::Never + | RigidTy::Foreign(_) + | RigidTy::Str => {} + RigidTy::Array(t, c) => { + *t = t.fold(folder)?; + *c = c.fold(folder)?; + } + RigidTy::Slice(inner) => *inner = inner.fold(folder)?, + RigidTy::RawPtr(ty, _) => *ty = ty.fold(folder)?, + RigidTy::Ref(_, ty, _) => *ty = ty.fold(folder)?, + RigidTy::FnDef(_, args) => *args = args.fold(folder)?, + RigidTy::FnPtr(sig) => *sig = sig.fold(folder)?, + RigidTy::Closure(_, args) => *args = args.fold(folder)?, + RigidTy::Generator(_, args, _) => *args = args.fold(folder)?, + RigidTy::Dynamic(pred, r, _) => { + *pred = pred.fold(folder)?; + *r = r.fold(folder)?; + } + RigidTy::Tuple(fields) => *fields = fields.fold(folder)?, + RigidTy::Adt(_, args) => *args = args.fold(folder)?, + } + ControlFlow::Continue(this) + } +} + +impl<T: Foldable> Foldable for Vec<T> { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut this = self.clone(); + for arg in &mut this { + *arg = arg.fold(folder)?; + } + ControlFlow::Continue(this) + } +} + +impl<T: Foldable> Foldable for Binder<T> { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(Self { + value: self.value.fold(folder)?, + bound_vars: self.bound_vars.clone(), + }) + } +} + +impl Foldable for ExistentialPredicate { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + let mut this = self.clone(); + match &mut this { + ExistentialPredicate::Trait(tr) => tr.generic_args = tr.generic_args.fold(folder)?, + ExistentialPredicate::Projection(p) => { + p.term = p.term.fold(folder)?; + p.generic_args = p.generic_args.fold(folder)?; + } + ExistentialPredicate::AutoTrait(_) => {} + } + ControlFlow::Continue(this) + } +} + +impl Foldable for TermKind { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(match self { + TermKind::Type(t) => TermKind::Type(t.fold(folder)?), + TermKind::Const(c) => TermKind::Const(c.fold(folder)?), + }) + } +} + +impl Foldable for FnSig { + fn super_fold<V: Folder>(&self, folder: &mut V) -> ControlFlow<V::Break, Self> { + ControlFlow::Continue(Self { + inputs_and_output: self.inputs_and_output.fold(folder)?, + c_variadic: self.c_variadic, + unsafety: self.unsafety, + abi: self.abi.clone(), + }) + } +} + +pub enum Never {} + +/// In order to instantiate a `Foldable`'s generic parameters with specific arguments, +/// `GenericArgs` can be used as a `Folder` that replaces all mentions of generic params +/// with the entries in its list. +impl Folder for GenericArgs { + type Break = Never; + + fn visit_ty(&mut self, ty: &Ty) -> ControlFlow<Self::Break, Ty> { + ControlFlow::Continue(match ty.kind() { + TyKind::Param(p) => self[p], + _ => *ty, + }) + } + + fn fold_const(&mut self, c: &Const) -> ControlFlow<Self::Break, Const> { + ControlFlow::Continue(match &c.literal { + ConstantKind::Param(p) => self[p.clone()].clone(), + _ => c.clone(), + }) + } +} diff --git a/compiler/rustc_smir/src/stable_mir/mir/body.rs b/compiler/rustc_smir/src/stable_mir/mir/body.rs index 72f719c2a5e..449ca4b8145 100644 --- a/compiler/rustc_smir/src/stable_mir/mir/body.rs +++ b/compiler/rustc_smir/src/stable_mir/mir/body.rs @@ -1,6 +1,6 @@ use crate::rustc_internal::Opaque; use crate::stable_mir::ty::{ - AdtDef, ClosureDef, Const, ConstantKind, GeneratorDef, GenericArgs, Movability, Region, + AdtDef, ClosureDef, Const, GeneratorDef, GenericArgs, Movability, Region, }; use crate::stable_mir::{self, ty::Ty, Span}; @@ -352,7 +352,7 @@ type UserTypeAnnotationIndex = usize; pub struct Constant { pub span: Span, pub user_ty: Option<UserTypeAnnotationIndex>, - pub literal: ConstantKind, + pub literal: Const, } #[derive(Clone, Debug)] @@ -391,7 +391,7 @@ pub enum Mutability { Mut, } -#[derive(Clone, Debug)] +#[derive(Copy, Clone, Debug)] pub enum Safety { Unsafe, Normal, diff --git a/compiler/rustc_smir/src/stable_mir/mod.rs b/compiler/rustc_smir/src/stable_mir/mod.rs index f9eafd9de7a..9cab7230b97 100644 --- a/compiler/rustc_smir/src/stable_mir/mod.rs +++ b/compiler/rustc_smir/src/stable_mir/mod.rs @@ -20,6 +20,7 @@ use self::ty::{ }; use crate::rustc_smir::Tables; +pub mod fold; pub mod mir; pub mod ty; pub mod visitor; @@ -86,7 +87,7 @@ pub struct CrateItem(pub(crate) DefId); impl CrateItem { pub fn body(&self) -> mir::Body { - with(|cx| cx.mir_body(self)) + with(|cx| cx.mir_body(self.0)) } } @@ -137,7 +138,7 @@ pub trait Context { fn entry_fn(&mut self) -> Option<CrateItem>; /// Retrieve all items of the local crate that have a MIR associated with them. fn all_local_items(&mut self) -> CrateItems; - fn mir_body(&mut self, item: &CrateItem) -> mir::Body; + fn mir_body(&mut self, item: DefId) -> mir::Body; fn all_trait_decls(&mut self) -> TraitDecls; fn trait_decl(&mut self, trait_def: &TraitDef) -> TraitDecl; fn all_trait_impls(&mut self) -> ImplTraitDecls; @@ -158,6 +159,9 @@ pub trait Context { /// Obtain the representation of a type. fn ty_kind(&mut self, ty: Ty) -> TyKind; + /// Create a new `Ty` from scratch without information from rustc. + fn mk_ty(&mut self, kind: TyKind) -> Ty; + /// HACK: Until we have fully stable consumers, we need an escape hatch /// to get `DefId`s out of `CrateItem`s. fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>)); diff --git a/compiler/rustc_smir/src/stable_mir/ty.rs b/compiler/rustc_smir/src/stable_mir/ty.rs index 1db6b1e3d28..54cba1263b7 100644 --- a/compiler/rustc_smir/src/stable_mir/ty.rs +++ b/compiler/rustc_smir/src/stable_mir/ty.rs @@ -1,18 +1,36 @@ -use super::{mir::Mutability, mir::Safety, with, AllocId, DefId}; +use super::{ + mir::Safety, + mir::{Body, Mutability}, + with, AllocId, DefId, +}; use crate::rustc_internal::Opaque; +use std::fmt::{self, Debug, Formatter}; -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone)] pub struct Ty(pub usize); +impl Debug for Ty { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("Ty").field("id", &self.0).field("kind", &self.kind()).finish() + } +} + impl Ty { pub fn kind(&self) -> TyKind { with(|context| context.ty_kind(*self)) } } +impl From<TyKind> for Ty { + fn from(value: TyKind) -> Self { + with(|context| context.mk_ty(value)) + } +} + #[derive(Debug, Clone)] pub struct Const { pub literal: ConstantKind, + pub ty: Ty, } type Ident = Opaque; @@ -88,6 +106,12 @@ pub struct ForeignDef(pub(crate) DefId); #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct FnDef(pub(crate) DefId); +impl FnDef { + pub fn body(&self) -> Body { + with(|ctx| ctx.mir_body(self.0)) + } +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ClosureDef(pub(crate) DefId); @@ -121,6 +145,22 @@ pub struct ImplDef(pub(crate) DefId); #[derive(Clone, Debug)] pub struct GenericArgs(pub Vec<GenericArgKind>); +impl std::ops::Index<ParamTy> for GenericArgs { + type Output = Ty; + + fn index(&self, index: ParamTy) -> &Self::Output { + self.0[index.index as usize].expect_ty() + } +} + +impl std::ops::Index<ParamConst> for GenericArgs { + type Output = Const; + + fn index(&self, index: ParamConst) -> &Self::Output { + self.0[index.index as usize].expect_const() + } +} + #[derive(Clone, Debug)] pub enum GenericArgKind { Lifetime(Region), @@ -128,6 +168,28 @@ pub enum GenericArgKind { Const(Const), } +impl GenericArgKind { + /// Panic if this generic argument is not a type, otherwise + /// return the type. + #[track_caller] + pub fn expect_ty(&self) -> &Ty { + match self { + GenericArgKind::Type(ty) => ty, + _ => panic!("{self:?}"), + } + } + + /// Panic if this generic argument is not a const, otherwise + /// return the const. + #[track_caller] + pub fn expect_const(&self) -> &Const { + match self { + GenericArgKind::Const(c) => c, + _ => panic!("{self:?}"), + } + } +} + #[derive(Clone, Debug)] pub enum TermKind { Type(Ty), @@ -287,12 +349,17 @@ pub struct Allocation { pub enum ConstantKind { Allocated(Allocation), Unevaluated(UnevaluatedConst), - ParamCt(Opaque), + Param(ParamConst), +} + +#[derive(Clone, Debug)] +pub struct ParamConst { + pub index: u32, + pub name: String, } #[derive(Clone, Debug)] pub struct UnevaluatedConst { - pub ty: Ty, pub def: ConstDef, pub args: GenericArgs, pub promoted: Option<Promoted>, diff --git a/compiler/rustc_smir/src/stable_mir/visitor.rs b/compiler/rustc_smir/src/stable_mir/visitor.rs index c928eb1381f..c86063d2ed6 100644 --- a/compiler/rustc_smir/src/stable_mir/visitor.rs +++ b/compiler/rustc_smir/src/stable_mir/visitor.rs @@ -30,11 +30,12 @@ impl Visitable for Ty { } fn super_visit<V: Visitor>(&self, visitor: &mut V) -> ControlFlow<V::Break> { match self.kind() { - super::ty::TyKind::RigidTy(ty) => ty.visit(visitor), - super::ty::TyKind::Alias(_, alias) => alias.args.visit(visitor), - super::ty::TyKind::Param(_) => todo!(), - super::ty::TyKind::Bound(_, _) => todo!(), + super::ty::TyKind::RigidTy(ty) => ty.visit(visitor)?, + super::ty::TyKind::Alias(_, alias) => alias.args.visit(visitor)?, + super::ty::TyKind::Param(_) => {} + super::ty::TyKind::Bound(_, _) => {} } + ControlFlow::Continue(()) } } @@ -44,10 +45,11 @@ impl Visitable for Const { } fn super_visit<V: Visitor>(&self, visitor: &mut V) -> ControlFlow<V::Break> { match &self.literal { - super::ty::ConstantKind::Allocated(alloc) => alloc.visit(visitor), - super::ty::ConstantKind::Unevaluated(uv) => uv.visit(visitor), - super::ty::ConstantKind::ParamCt(param) => param.visit(visitor), + super::ty::ConstantKind::Allocated(alloc) => alloc.visit(visitor)?, + super::ty::ConstantKind::Unevaluated(uv) => uv.visit(visitor)?, + super::ty::ConstantKind::Param(_) => {} } + self.ty.visit(visitor) } } @@ -65,8 +67,7 @@ impl Visitable for Allocation { impl Visitable for UnevaluatedConst { fn super_visit<V: Visitor>(&self, visitor: &mut V) -> ControlFlow<V::Break> { - let UnevaluatedConst { ty, def, args, promoted } = self; - ty.visit(visitor)?; + let UnevaluatedConst { def, args, promoted } = self; def.visit(visitor)?; args.visit(visitor)?; promoted.visit(visitor) diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index f1a6e9059d7..3a869a2410a 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -33,7 +33,7 @@ extern crate rustc_macros; #[macro_use] extern crate tracing; -use rustc_data_structures::AtomicRef; +use rustc_data_structures::{cold_path, AtomicRef}; use rustc_macros::HashStable_Generic; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; @@ -64,7 +64,7 @@ pub mod fatal_error; pub mod profiling; use rustc_data_structures::stable_hasher::{Hash128, Hash64, HashStable, StableHasher}; -use rustc_data_structures::sync::{Lock, Lrc}; +use rustc_data_structures::sync::{FreezeLock, FreezeWriteGuard, Lock, Lrc}; use std::borrow::Cow; use std::cmp::{self, Ordering}; @@ -1206,7 +1206,6 @@ pub enum ExternalSourceKind { AbsentOk, /// A failed attempt has been made to load the external source. AbsentErr, - Unneeded, } impl ExternalSource { @@ -1343,13 +1342,13 @@ pub struct SourceFile { pub src_hash: SourceFileHash, /// The external source code (used for external crates, which will have a `None` /// value as `self.src`. - pub external_src: Lock<ExternalSource>, + pub external_src: FreezeLock<ExternalSource>, /// The start position of this source in the `SourceMap`. pub start_pos: BytePos, /// The byte length of this source. pub source_len: RelativeBytePos, /// Locations of lines beginnings in the source code. - pub lines: Lock<SourceFileLines>, + pub lines: FreezeLock<SourceFileLines>, /// Locations of multi-byte characters in the source code. pub multibyte_chars: Vec<MultiByteChar>, /// Width of characters that are not narrow in the source code. @@ -1368,10 +1367,10 @@ impl Clone for SourceFile { name: self.name.clone(), src: self.src.clone(), src_hash: self.src_hash, - external_src: Lock::new(self.external_src.borrow().clone()), + external_src: self.external_src.clone(), start_pos: self.start_pos, source_len: self.source_len, - lines: Lock::new(self.lines.borrow().clone()), + lines: self.lines.clone(), multibyte_chars: self.multibyte_chars.clone(), non_narrow_chars: self.non_narrow_chars.clone(), normalized_pos: self.normalized_pos.clone(), @@ -1389,64 +1388,63 @@ impl<S: Encoder> Encodable<S> for SourceFile { self.source_len.encode(s); // We are always in `Lines` form by the time we reach here. - assert!(self.lines.borrow().is_lines()); - self.lines(|lines| { - // Store the length. - s.emit_u32(lines.len() as u32); - - // Compute and store the difference list. - if lines.len() != 0 { - let max_line_length = if lines.len() == 1 { - 0 - } else { - lines - .array_windows() - .map(|&[fst, snd]| snd - fst) - .map(|bp| bp.to_usize()) - .max() - .unwrap() - }; - - let bytes_per_diff: usize = match max_line_length { - 0..=0xFF => 1, - 0x100..=0xFFFF => 2, - _ => 4, - }; - - // Encode the number of bytes used per diff. - s.emit_u8(bytes_per_diff as u8); - - // Encode the first element. - assert_eq!(lines[0], RelativeBytePos(0)); - - // Encode the difference list. - let diff_iter = lines.array_windows().map(|&[fst, snd]| snd - fst); - let num_diffs = lines.len() - 1; - let mut raw_diffs; - match bytes_per_diff { - 1 => { - raw_diffs = Vec::with_capacity(num_diffs); - for diff in diff_iter { - raw_diffs.push(diff.0 as u8); - } + assert!(self.lines.read().is_lines()); + let lines = self.lines(); + // Store the length. + s.emit_u32(lines.len() as u32); + + // Compute and store the difference list. + if lines.len() != 0 { + let max_line_length = if lines.len() == 1 { + 0 + } else { + lines + .array_windows() + .map(|&[fst, snd]| snd - fst) + .map(|bp| bp.to_usize()) + .max() + .unwrap() + }; + + let bytes_per_diff: usize = match max_line_length { + 0..=0xFF => 1, + 0x100..=0xFFFF => 2, + _ => 4, + }; + + // Encode the number of bytes used per diff. + s.emit_u8(bytes_per_diff as u8); + + // Encode the first element. + assert_eq!(lines[0], RelativeBytePos(0)); + + // Encode the difference list. + let diff_iter = lines.array_windows().map(|&[fst, snd]| snd - fst); + let num_diffs = lines.len() - 1; + let mut raw_diffs; + match bytes_per_diff { + 1 => { + raw_diffs = Vec::with_capacity(num_diffs); + for diff in diff_iter { + raw_diffs.push(diff.0 as u8); } - 2 => { - raw_diffs = Vec::with_capacity(bytes_per_diff * num_diffs); - for diff in diff_iter { - raw_diffs.extend_from_slice(&(diff.0 as u16).to_le_bytes()); - } + } + 2 => { + raw_diffs = Vec::with_capacity(bytes_per_diff * num_diffs); + for diff in diff_iter { + raw_diffs.extend_from_slice(&(diff.0 as u16).to_le_bytes()); } - 4 => { - raw_diffs = Vec::with_capacity(bytes_per_diff * num_diffs); - for diff in diff_iter { - raw_diffs.extend_from_slice(&(diff.0).to_le_bytes()); - } + } + 4 => { + raw_diffs = Vec::with_capacity(bytes_per_diff * num_diffs); + for diff in diff_iter { + raw_diffs.extend_from_slice(&(diff.0).to_le_bytes()); } - _ => unreachable!(), } - s.emit_raw_bytes(&raw_diffs); + _ => unreachable!(), } - }); + s.emit_raw_bytes(&raw_diffs); + } self.multibyte_chars.encode(s); self.non_narrow_chars.encode(s); @@ -1488,8 +1486,8 @@ impl<D: Decoder> Decodable<D> for SourceFile { src_hash, // Unused - the metadata decoder will construct // a new SourceFile, filling in `external_src` properly - external_src: Lock::new(ExternalSource::Unneeded), - lines: Lock::new(lines), + external_src: FreezeLock::frozen(ExternalSource::Unneeded), + lines: FreezeLock::new(lines), multibyte_chars, non_narrow_chars, normalized_pos, @@ -1530,10 +1528,10 @@ impl SourceFile { name, src: Some(Lrc::new(src)), src_hash, - external_src: Lock::new(ExternalSource::Unneeded), + external_src: FreezeLock::frozen(ExternalSource::Unneeded), start_pos: BytePos::from_u32(0), source_len: RelativeBytePos::from_u32(source_len), - lines: Lock::new(SourceFileLines::Lines(lines)), + lines: FreezeLock::frozen(SourceFileLines::Lines(lines)), multibyte_chars, non_narrow_chars, normalized_pos, @@ -1542,65 +1540,82 @@ impl SourceFile { }) } - pub fn lines<F, R>(&self, f: F) -> R - where - F: FnOnce(&[RelativeBytePos]) -> R, - { - let mut guard = self.lines.borrow_mut(); - match &*guard { - SourceFileLines::Lines(lines) => f(lines), - SourceFileLines::Diffs(SourceFileDiffs { bytes_per_diff, num_diffs, raw_diffs }) => { - // Convert from "diffs" form to "lines" form. - let num_lines = num_diffs + 1; - let mut lines = Vec::with_capacity(num_lines); - let mut line_start = RelativeBytePos(0); - lines.push(line_start); - - assert_eq!(*num_diffs, raw_diffs.len() / bytes_per_diff); - match bytes_per_diff { - 1 => { - lines.extend(raw_diffs.into_iter().map(|&diff| { - line_start = line_start + RelativeBytePos(diff as u32); - line_start - })); - } - 2 => { - lines.extend((0..*num_diffs).map(|i| { - let pos = bytes_per_diff * i; - let bytes = [raw_diffs[pos], raw_diffs[pos + 1]]; - let diff = u16::from_le_bytes(bytes); - line_start = line_start + RelativeBytePos(diff as u32); - line_start - })); - } - 4 => { - lines.extend((0..*num_diffs).map(|i| { - let pos = bytes_per_diff * i; - let bytes = [ - raw_diffs[pos], - raw_diffs[pos + 1], - raw_diffs[pos + 2], - raw_diffs[pos + 3], - ]; - let diff = u32::from_le_bytes(bytes); - line_start = line_start + RelativeBytePos(diff); - line_start - })); - } - _ => unreachable!(), - } - let res = f(&lines); - *guard = SourceFileLines::Lines(lines); - res + /// This converts the `lines` field to contain `SourceFileLines::Lines` if needed and freezes it. + fn convert_diffs_to_lines_frozen(&self) { + let mut guard = if let Some(guard) = self.lines.try_write() { guard } else { return }; + + let SourceFileDiffs { bytes_per_diff, num_diffs, raw_diffs } = match &*guard { + SourceFileLines::Diffs(diffs) => diffs, + SourceFileLines::Lines(..) => { + FreezeWriteGuard::freeze(guard); + return; + } + }; + + // Convert from "diffs" form to "lines" form. + let num_lines = num_diffs + 1; + let mut lines = Vec::with_capacity(num_lines); + let mut line_start = RelativeBytePos(0); + lines.push(line_start); + + assert_eq!(*num_diffs, raw_diffs.len() / bytes_per_diff); + match bytes_per_diff { + 1 => { + lines.extend(raw_diffs.into_iter().map(|&diff| { + line_start = line_start + RelativeBytePos(diff as u32); + line_start + })); + } + 2 => { + lines.extend((0..*num_diffs).map(|i| { + let pos = bytes_per_diff * i; + let bytes = [raw_diffs[pos], raw_diffs[pos + 1]]; + let diff = u16::from_le_bytes(bytes); + line_start = line_start + RelativeBytePos(diff as u32); + line_start + })); } + 4 => { + lines.extend((0..*num_diffs).map(|i| { + let pos = bytes_per_diff * i; + let bytes = [ + raw_diffs[pos], + raw_diffs[pos + 1], + raw_diffs[pos + 2], + raw_diffs[pos + 3], + ]; + let diff = u32::from_le_bytes(bytes); + line_start = line_start + RelativeBytePos(diff); + line_start + })); + } + _ => unreachable!(), } + + *guard = SourceFileLines::Lines(lines); + + FreezeWriteGuard::freeze(guard); + } + + pub fn lines(&self) -> &[RelativeBytePos] { + if let Some(SourceFileLines::Lines(lines)) = self.lines.get() { + return &lines[..]; + } + + cold_path(|| { + self.convert_diffs_to_lines_frozen(); + if let Some(SourceFileLines::Lines(lines)) = self.lines.get() { + return &lines[..]; + } + unreachable!() + }) } /// Returns the `BytePos` of the beginning of the current line. pub fn line_begin_pos(&self, pos: BytePos) -> BytePos { let pos = self.relative_position(pos); let line_index = self.lookup_line(pos).unwrap(); - let line_start_pos = self.lines(|lines| lines[line_index]); + let line_start_pos = self.lines()[line_index]; self.absolute_position(line_start_pos) } @@ -1612,35 +1627,37 @@ impl SourceFile { where F: FnOnce() -> Option<String>, { - if matches!( - *self.external_src.borrow(), - ExternalSource::Foreign { kind: ExternalSourceKind::AbsentOk, .. } - ) { + if !self.external_src.is_frozen() { let src = get_src(); - let mut external_src = self.external_src.borrow_mut(); - // Check that no-one else have provided the source while we were getting it - if let ExternalSource::Foreign { - kind: src_kind @ ExternalSourceKind::AbsentOk, .. - } = &mut *external_src - { - if let Some(mut src) = src { - // The src_hash needs to be computed on the pre-normalized src. - if self.src_hash.matches(&src) { - normalize_src(&mut src); - *src_kind = ExternalSourceKind::Present(Lrc::new(src)); - return true; - } + let src = src.and_then(|mut src| { + // The src_hash needs to be computed on the pre-normalized src. + self.src_hash.matches(&src).then(|| { + normalize_src(&mut src); + src + }) + }); + + self.external_src.try_write().map(|mut external_src| { + if let ExternalSource::Foreign { + kind: src_kind @ ExternalSourceKind::AbsentOk, + .. + } = &mut *external_src + { + *src_kind = if let Some(src) = src { + ExternalSourceKind::Present(Lrc::new(src)) + } else { + ExternalSourceKind::AbsentErr + }; } else { - *src_kind = ExternalSourceKind::AbsentErr; + panic!("unexpected state {:?}", *external_src) } - false - } else { - self.src.is_some() || external_src.get_source().is_some() - } - } else { - self.src.is_some() || self.external_src.borrow().get_source().is_some() + // Freeze this so we don't try to load the source again. + FreezeWriteGuard::freeze(external_src) + }); } + + self.src.is_some() || self.external_src.read().get_source().is_some() } /// Gets a line from the list of pre-computed line-beginnings. @@ -1658,7 +1675,7 @@ impl SourceFile { } let begin = { - let line = self.lines(|lines| lines.get(line_number).copied())?; + let line = self.lines().get(line_number).copied()?; line.to_usize() }; @@ -1682,7 +1699,7 @@ impl SourceFile { } pub fn count_lines(&self) -> usize { - self.lines(|lines| lines.len()) + self.lines().len() } #[inline] @@ -1705,7 +1722,7 @@ impl SourceFile { /// number. If the source_file is empty or the position is located before the /// first line, `None` is returned. pub fn lookup_line(&self, pos: RelativeBytePos) -> Option<usize> { - self.lines(|lines| lines.partition_point(|x| x <= &pos).checked_sub(1)) + self.lines().partition_point(|x| x <= &pos).checked_sub(1) } pub fn line_bounds(&self, line_index: usize) -> Range<BytePos> { @@ -1713,15 +1730,13 @@ impl SourceFile { return self.start_pos..self.start_pos; } - self.lines(|lines| { - assert!(line_index < lines.len()); - if line_index == (lines.len() - 1) { - self.absolute_position(lines[line_index])..self.end_position() - } else { - self.absolute_position(lines[line_index]) - ..self.absolute_position(lines[line_index + 1]) - } - }) + let lines = self.lines(); + assert!(line_index < lines.len()); + if line_index == (lines.len() - 1) { + self.absolute_position(lines[line_index])..self.end_position() + } else { + self.absolute_position(lines[line_index])..self.absolute_position(lines[line_index + 1]) + } } /// Returns whether or not the file contains the given `SourceMap` byte @@ -1807,7 +1822,7 @@ impl SourceFile { match self.lookup_line(pos) { Some(a) => { let line = a + 1; // Line numbers start at 1 - let linebpos = self.lines(|lines| lines[a]); + let linebpos = self.lines()[a]; let linechpos = self.bytepos_to_file_charpos(linebpos); let col = chpos - linechpos; debug!("byte pos {:?} is on the line at byte pos {:?}", pos, linebpos); @@ -1827,7 +1842,7 @@ impl SourceFile { let (line, col_or_chpos) = self.lookup_file_pos(pos); if line > 0 { let col = col_or_chpos; - let linebpos = self.lines(|lines| lines[line - 1]); + let linebpos = self.lines()[line - 1]; let col_display = { let start_width_idx = self .non_narrow_chars diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs index 81730f2f608..68727a6c40e 100644 --- a/compiler/rustc_span/src/source_map.rs +++ b/compiler/rustc_span/src/source_map.rs @@ -328,7 +328,7 @@ impl SourceMap { name_hash: Hash128, source_len: u32, cnum: CrateNum, - file_local_lines: Lock<SourceFileLines>, + file_local_lines: FreezeLock<SourceFileLines>, multibyte_chars: Vec<MultiByteChar>, non_narrow_chars: Vec<NonNarrowChar>, normalized_pos: Vec<NormalizedPos>, @@ -340,7 +340,7 @@ impl SourceMap { name: filename, src: None, src_hash, - external_src: Lock::new(ExternalSource::Foreign { + external_src: FreezeLock::new(ExternalSource::Foreign { kind: ExternalSourceKind::AbsentOk, metadata_index, }), @@ -564,7 +564,7 @@ impl SourceMap { end: (local_end.sf.name.clone(), local_end.sf.start_pos), }))) } else { - self.ensure_source_file_source_present(local_begin.sf.clone()); + self.ensure_source_file_source_present(&local_begin.sf); let start_index = local_begin.pos.to_usize(); let end_index = local_end.pos.to_usize(); @@ -581,7 +581,7 @@ impl SourceMap { if let Some(ref src) = local_begin.sf.src { extract_source(src, start_index, end_index) - } else if let Some(src) = local_begin.sf.external_src.borrow().get_source() { + } else if let Some(src) = local_begin.sf.external_src.read().get_source() { extract_source(src, start_index, end_index) } else { Err(SpanSnippetError::SourceNotAvailable { filename: local_begin.sf.name.clone() }) @@ -873,7 +873,7 @@ impl SourceMap { let sp = sp.data(); let local_begin = self.lookup_byte_offset(sp.lo); let start_index = local_begin.pos.to_usize(); - let src = local_begin.sf.external_src.borrow(); + let src = local_begin.sf.external_src.read(); let snippet = if let Some(ref src) = local_begin.sf.src { Some(&src[start_index..]) @@ -983,7 +983,7 @@ impl SourceMap { return 1; } - let src = local_begin.sf.external_src.borrow(); + let src = local_begin.sf.external_src.read(); let snippet = if let Some(src) = &local_begin.sf.src { src @@ -1030,7 +1030,7 @@ impl SourceMap { self.files().iter().fold(0, |a, f| a + f.count_lines()) } - pub fn ensure_source_file_source_present(&self, source_file: Lrc<SourceFile>) -> bool { + pub fn ensure_source_file_source_present(&self, source_file: &SourceFile) -> bool { source_file.add_external_src(|| { let FileName::Real(ref name) = source_file.name else { return None; diff --git a/compiler/rustc_span/src/source_map/tests.rs b/compiler/rustc_span/src/source_map/tests.rs index 7689e6afac5..e393db02064 100644 --- a/compiler/rustc_span/src/source_map/tests.rs +++ b/compiler/rustc_span/src/source_map/tests.rs @@ -1,6 +1,6 @@ use super::*; -use rustc_data_structures::sync::Lrc; +use rustc_data_structures::sync::{FreezeLock, Lrc}; fn init_source_map() -> SourceMap { let sm = SourceMap::new(FilePathMapping::empty()); @@ -246,7 +246,7 @@ fn t10() { name_hash, source_len.to_u32(), CrateNum::new(0), - lines, + FreezeLock::new(lines.read().clone()), multibyte_chars, non_narrow_chars, normalized_pos, diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 656deebb5d0..3247d5e46ef 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -387,6 +387,7 @@ symbols! { asm_sym, asm_unwind, assert, + assert_eq, assert_eq_macro, assert_inhabited, assert_macro, @@ -1639,6 +1640,7 @@ symbols! { unsafe_block_in_unsafe_fn, unsafe_cell, unsafe_cell_from_mut, + unsafe_cell_raw_get, unsafe_no_drop_flag, unsafe_pin_internals, unsize, diff --git a/compiler/rustc_span/src/tests.rs b/compiler/rustc_span/src/tests.rs index a980ee8d9e0..cb88fa89058 100644 --- a/compiler/rustc_span/src/tests.rs +++ b/compiler/rustc_span/src/tests.rs @@ -7,9 +7,7 @@ fn test_lookup_line() { SourceFile::new(FileName::Anon(Hash64::ZERO), source, SourceFileHashAlgorithm::Sha256) .unwrap(); sf.start_pos = BytePos(3); - sf.lines(|lines| { - assert_eq!(lines, &[RelativeBytePos(0), RelativeBytePos(14), RelativeBytePos(25)]) - }); + assert_eq!(sf.lines(), &[RelativeBytePos(0), RelativeBytePos(14), RelativeBytePos(25)]); assert_eq!(sf.lookup_line(RelativeBytePos(0)), Some(0)); assert_eq!(sf.lookup_line(RelativeBytePos(1)), Some(0)); diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 8d573def9bb..5d75974279e 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -36,7 +36,10 @@ pub enum PassMode { Ignore, /// Pass the argument directly. /// - /// The argument has a layout abi of `Scalar`, `Vector` or in rare cases `Aggregate`. + /// The argument has a layout abi of `Scalar` or `Vector`. + /// Unfortunately due to past mistakes, in rare cases on wasm, it can also be `Aggregate`. + /// This is bad since it leaks LLVM implementation details into the ABI. + /// (Also see <https://github.com/rust-lang/rust/issues/115666>.) Direct(ArgAttributes), /// Pass a pair's elements directly in two arguments. /// @@ -55,6 +58,29 @@ pub enum PassMode { Indirect { attrs: ArgAttributes, extra_attrs: Option<ArgAttributes>, on_stack: bool }, } +impl PassMode { + /// Checks if these two `PassMode` are equal enough to be considered "the same for all + /// function call ABIs". However, the `Layout` can also impact ABI decisions, + /// so that needs to be compared as well! + pub fn eq_abi(&self, other: &Self) -> bool { + match (self, other) { + (PassMode::Ignore, PassMode::Ignore) => true, // can still be reached for the return type + (PassMode::Direct(a1), PassMode::Direct(a2)) => a1.eq_abi(a2), + (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => a1.eq_abi(a2) && b1.eq_abi(b2), + (PassMode::Cast(c1, pad1), PassMode::Cast(c2, pad2)) => c1.eq_abi(c2) && pad1 == pad2, + ( + PassMode::Indirect { attrs: a1, extra_attrs: None, on_stack: s1 }, + PassMode::Indirect { attrs: a2, extra_attrs: None, on_stack: s2 }, + ) => a1.eq_abi(a2) && s1 == s2, + ( + PassMode::Indirect { attrs: a1, extra_attrs: Some(e1), on_stack: s1 }, + PassMode::Indirect { attrs: a2, extra_attrs: Some(e2), on_stack: s2 }, + ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2, + _ => false, + } + } +} + // Hack to disable non_upper_case_globals only for the bitflags! and not for the rest // of this module pub use attr_impl::ArgAttribute; @@ -127,6 +153,24 @@ impl ArgAttributes { pub fn contains(&self, attr: ArgAttribute) -> bool { self.regular.contains(attr) } + + /// Checks if these two `ArgAttributes` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool { + // There's only one regular attribute that matters for the call ABI: InReg. + // Everything else is things like noalias, dereferenceable, nonnull, ... + // (This also applies to pointee_size, pointee_align.) + if self.regular.contains(ArgAttribute::InReg) != other.regular.contains(ArgAttribute::InReg) + { + return false; + } + // We also compare the sign extension mode -- this could let the callee make assumptions + // about bits that conceptually were not even passed. + if self.arg_ext != other.arg_ext { + return false; + } + return true; + } } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] @@ -272,6 +316,14 @@ impl CastTarget { acc.max(align) }) } + + /// Checks if these two `CastTarget` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool { + let CastTarget { prefix: prefix_l, rest: rest_l, attrs: attrs_l } = self; + let CastTarget { prefix: prefix_r, rest: rest_r, attrs: attrs_r } = other; + prefix_l == prefix_r && rest_l == rest_r && attrs_l.eq_abi(attrs_r) + } } /// Return value from the `homogeneous_aggregate` test function. @@ -465,6 +517,7 @@ pub struct ArgAbi<'a, Ty> { } impl<'a, Ty> ArgAbi<'a, Ty> { + /// This defines the "default ABI" for that type, that is then later adjusted in `fn_abi_adjust_for_abi`. pub fn new( cx: &impl HasDataLayout, layout: TyAndLayout<'a, Ty>, @@ -478,6 +531,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> { scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)), ), Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), + // The `Aggregate` ABI should always be adjusted later. Abi::Aggregate { .. } => PassMode::Direct(ArgAttributes::new()), }; ArgAbi { layout, mode } @@ -570,6 +624,14 @@ impl<'a, Ty> ArgAbi<'a, Ty> { pub fn is_ignore(&self) -> bool { matches!(self.mode, PassMode::Ignore) } + + /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all + /// function call ABIs". + pub fn eq_abi(&self, other: &Self) -> bool { + // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look + // at the type. + self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode) + } } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs index 0eb2309ecb2..796b752ff9d 100644 --- a/compiler/rustc_target/src/abi/call/wasm.rs +++ b/compiler/rustc_target/src/abi/call/wasm.rs @@ -61,6 +61,10 @@ where /// The purpose of this ABI is for matching the WebAssembly standard. This /// intentionally diverges from the C ABI and is specifically crafted to take /// advantage of LLVM's support of multiple returns in WebAssembly. +/// +/// This ABI is *bad*! It uses `PassMode::Direct` for `abi::Aggregate` types, which leaks LLVM +/// implementation details into the ABI. It's just hard to fix because ABIs are hard to change. +/// Also see <https://github.com/rust-lang/rust/issues/115666>. pub fn compute_wasm_abi_info<Ty>(fn_abi: &mut FnAbi<'_, Ty>) { if !fn_abi.ret.is_ignore() { classify_ret(&mut fn_abi.ret); diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs index 611ec6b00ef..32fb10ce4a6 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs @@ -838,7 +838,20 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> { obligation.param_env, real_trait_pred_and_base_ty, ); - if self.predicate_may_hold(&obligation) { + let sized_obligation = Obligation::new( + self.tcx, + obligation.cause.clone(), + obligation.param_env, + ty::TraitRef::from_lang_item( + self.tcx, + hir::LangItem::Sized, + obligation.cause.span, + [base_ty], + ), + ); + if self.predicate_may_hold(&obligation) + && self.predicate_must_hold_modulo_regions(&sized_obligation) + { let call_node = self.tcx.hir().get(*call_hir_id); let msg = "consider dereferencing here"; let is_receiver = matches!( diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 802391f1aad..3ffe670d87a 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -520,6 +520,8 @@ fn fn_abi_adjust_for_abi<'tcx>( _ => return, } + // `Aggregate` ABI must be adjusted to ensure that ABI-compatible Rust types are passed + // the same way. let size = arg.layout.size; if arg.layout.is_unsized() || size > Pointer(AddressSpace::DATA).size(cx) { diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 34fd31e49e1..1fc5d9359a4 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -19,13 +19,32 @@ fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx> // needs drop. let adt_has_dtor = |adt_def: ty::AdtDef<'tcx>| adt_def.destructor(tcx).map(|_| DtorType::Significant); - let res = - drop_tys_helper(tcx, query.value, query.param_env, adt_has_dtor, false).next().is_some(); + let res = drop_tys_helper(tcx, query.value, query.param_env, adt_has_dtor, false) + .filter(filter_array_elements(tcx, query.param_env)) + .next() + .is_some(); debug!("needs_drop_raw({:?}) = {:?}", query, res); res } +/// HACK: in order to not mistakenly assume that `[PhantomData<T>; N]` requires drop glue +/// we check the element type for drop glue. The correct fix would be looking at the +/// entirety of the code around `needs_drop_components` and this file and come up with +/// logic that is easier to follow while not repeating any checks that may thus diverge. +fn filter_array_elements<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, +) -> impl Fn(&Result<Ty<'tcx>, AlwaysRequiresDrop>) -> bool { + move |ty| match ty { + Ok(ty) => match *ty.kind() { + ty::Array(elem, _) => tcx.needs_drop_raw(param_env.and(elem)), + _ => true, + }, + Err(AlwaysRequiresDrop) => true, + } +} + fn has_significant_drop_raw<'tcx>( tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, @@ -37,6 +56,7 @@ fn has_significant_drop_raw<'tcx>( adt_consider_insignificant_dtor(tcx), true, ) + .filter(filter_array_elements(tcx, query.param_env)) .next() .is_some(); debug!("has_significant_drop_raw({:?}) = {:?}", query, res); diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs index e0abc7f04f5..e348591ebba 100644 --- a/compiler/rustc_type_ir/src/lib.rs +++ b/compiler/rustc_type_ir/src/lib.rs @@ -574,16 +574,16 @@ rustc_index::newtype_index! { pub struct TyVid {} } -/// An **int**egral (`u32`, `i32`, `usize`, etc.) type **v**ariable **ID**. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] -pub struct IntVid { - pub index: u32, +rustc_index::newtype_index! { + /// An **int**egral (`u32`, `i32`, `usize`, etc.) type **v**ariable **ID**. + #[debug_format = "?{}i"] + pub struct IntVid {} } -/// An **float**ing-point (`f32` or `f64`) type **v**ariable **ID**. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)] -pub struct FloatVid { - pub index: u32, +rustc_index::newtype_index! { + /// A **float**ing-point (`f32` or `f64`) type **v**ariable **ID**. + #[debug_format = "?{}f"] + pub struct FloatVid {} } /// A placeholder for a type that hasn't been inferred yet. @@ -645,11 +645,11 @@ impl UnifyKey for IntVid { type Value = Option<IntVarValue>; #[inline] // make this function eligible for inlining - it is quite hot. fn index(&self) -> u32 { - self.index + self.as_u32() } #[inline] fn from_index(i: u32) -> IntVid { - IntVid { index: i } + IntVid::from_u32(i) } fn tag() -> &'static str { "IntVid" @@ -662,11 +662,11 @@ impl UnifyKey for FloatVid { type Value = Option<FloatVarValue>; #[inline] fn index(&self) -> u32 { - self.index + self.as_u32() } #[inline] fn from_index(i: u32) -> FloatVid { - FloatVid { index: i } + FloatVid::from_u32(i) } fn tag() -> &'static str { "FloatVid" @@ -770,18 +770,6 @@ impl fmt::Debug for FloatVarValue { } } -impl fmt::Debug for IntVid { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "?{}i", self.index) - } -} - -impl fmt::Debug for FloatVid { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "?{}f", self.index) - } -} - impl fmt::Debug for Variance { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match *self { |