diff options
Diffstat (limited to 'compiler')
116 files changed, 2321 insertions, 2044 deletions
diff --git a/compiler/rustc_borrowck/src/def_use.rs b/compiler/rustc_borrowck/src/def_use.rs index b775739fed2..b719a610e07 100644 --- a/compiler/rustc_borrowck/src/def_use.rs +++ b/compiler/rustc_borrowck/src/def_use.rs @@ -50,7 +50,6 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> { PlaceContext::MutatingUse(MutatingUseContext::Borrow) | PlaceContext::NonMutatingUse(NonMutatingUseContext::SharedBorrow) | PlaceContext::NonMutatingUse(NonMutatingUseContext::ShallowBorrow) | - PlaceContext::NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | // `PlaceMention` and `AscribeUserType` both evaluate the place, which must not // contain dangling references. diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs index f41795d60a0..cfcf31fce32 100644 --- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs @@ -180,24 +180,25 @@ trait TypeOpInfo<'tcx> { return; }; - let placeholder_region = tcx.mk_re_placeholder(ty::Placeholder { - universe: adjusted_universe.into(), - bound: placeholder.bound, - }); - - let error_region = - if let RegionElement::PlaceholderRegion(error_placeholder) = error_element { - let adjusted_universe = - error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32()); - adjusted_universe.map(|adjusted| { - tcx.mk_re_placeholder(ty::Placeholder { - universe: adjusted.into(), - bound: error_placeholder.bound, - }) - }) - } else { - None - }; + let placeholder_region = ty::Region::new_placeholder( + tcx, + ty::Placeholder { universe: adjusted_universe.into(), bound: placeholder.bound }, + ); + + let error_region = if let RegionElement::PlaceholderRegion(error_placeholder) = + error_element + { + let adjusted_universe = + error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32()); + adjusted_universe.map(|adjusted| { + ty::Region::new_placeholder( + tcx, + ty::Placeholder { universe: adjusted.into(), bound: error_placeholder.bound }, + ) + }) + } else { + None + }; debug!(?placeholder_region); @@ -390,7 +391,7 @@ fn try_extract_error_from_fulfill_cx<'tcx>( error_region, ®ion_constraints, |vid| ocx.infcx.region_var_origin(vid), - |vid| ocx.infcx.universe_of_region(ocx.infcx.tcx.mk_re_var(vid)), + |vid| ocx.infcx.universe_of_region(ty::Region::new_var(ocx.infcx.tcx, vid)), ) } @@ -411,7 +412,7 @@ fn try_extract_error_from_region_constraints<'tcx>( } // FIXME: Should this check the universe of the var? Constraint::VarSubReg(vid, sup) if sup == placeholder_region => { - Some((infcx.tcx.mk_re_var(vid), cause.clone())) + Some((ty::Region::new_var(infcx.tcx, vid), cause.clone())) } _ => None, } diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 889acb3acbe..b5014a3f479 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -441,7 +441,7 @@ fn for_each_region_constraint<'tcx>( let subject = match req.subject { ClosureOutlivesSubject::Region(subject) => format!("{:?}", subject), ClosureOutlivesSubject::Ty(ty) => { - format!("{:?}", ty.instantiate(tcx, |vid| tcx.mk_re_var(vid))) + format!("{:?}", ty.instantiate(tcx, |vid| ty::Region::new_var(tcx, vid))) } }; with_msg(format!("where {}: {:?}", subject, req.outlived_free_region,))?; diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs index 50b246b1478..2a0cb49672b 100644 --- a/compiler/rustc_borrowck/src/region_infer/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/mod.rs @@ -1158,7 +1158,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { .universal_regions_outlived_by(r_scc) .filter(|&u_r| !self.universal_regions.is_local_free_region(u_r)) .find(|&u_r| self.eval_equal(u_r, r_vid)) - .map(|u_r| tcx.mk_re_var(u_r)) + .map(|u_r| ty::Region::new_var(tcx, u_r)) // In the case of a failure, use `ReErased`. We will eventually // return `None` in this case. .unwrap_or(tcx.lifetimes.re_erased) @@ -1355,7 +1355,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { let vid = self.to_region_vid(r); let scc = self.constraint_sccs.scc(vid); let repr = self.scc_representatives[scc]; - tcx.mk_re_var(repr) + ty::Region::new_var(tcx, repr) }) } @@ -1779,7 +1779,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { } // If not, report an error. - let member_region = infcx.tcx.mk_re_var(member_region_vid); + let member_region = ty::Region::new_var(infcx.tcx, member_region_vid); errors_buffer.push(RegionErrorKind::UnexpectedHiddenRegion { span: m_c.definition_span, hidden_ty: m_c.hidden_ty, diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs index 7fc89e89a35..33f07c1d8fc 100644 --- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs @@ -92,7 +92,8 @@ impl<'tcx> RegionInferenceContext<'tcx> { } None => { subst_regions.push(vid); - infcx.tcx.mk_re_error_with_message( + ty::Region::new_error_with_message( + infcx.tcx, concrete_type.span, "opaque type with non-universal region substs", ) diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 51a84ce6cad..3b896f6540c 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -139,7 +139,7 @@ pub(crate) fn type_check<'mir, 'tcx>( upvars: &[Upvar<'tcx>], use_polonius: bool, ) -> MirTypeckResults<'tcx> { - let implicit_region_bound = infcx.tcx.mk_re_var(universal_regions.fr_fn_body); + let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body); let mut constraints = MirTypeckRegionConstraints { placeholder_indices: PlaceholderIndices::default(), placeholder_index_to_region: IndexVec::default(), @@ -766,8 +766,8 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { PlaceContext::MutatingUse(_) => ty::Invariant, PlaceContext::NonUse(StorageDead | StorageLive | VarDebugInfo) => ty::Invariant, PlaceContext::NonMutatingUse( - Inspect | Copy | Move | PlaceMention | SharedBorrow | ShallowBorrow | UniqueBorrow - | AddressOf | Projection, + Inspect | Copy | Move | PlaceMention | SharedBorrow | ShallowBorrow | AddressOf + | Projection, ) => ty::Covariant, PlaceContext::NonUse(AscribeUserTy(variance)) => variance, } diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs index 56f078f2da8..c871703429a 100644 --- a/compiler/rustc_borrowck/src/universal_regions.rs +++ b/compiler/rustc_borrowck/src/universal_regions.rs @@ -500,7 +500,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("c-variadic"))) .as_var(); - let region = self.infcx.tcx.mk_re_var(reg_vid); + let region = ty::Region::new_var(self.infcx.tcx, reg_vid); let va_list_ty = self.infcx.tcx.type_of(va_list_did).subst(self.infcx.tcx, &[region.into()]); @@ -660,7 +660,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BrEnv, }; - let env_region = tcx.mk_re_late_bound(ty::INNERMOST, br); + let env_region = ty::Region::new_late_bound(tcx, ty::INNERMOST, br); let closure_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap(); // The "inputs" of the closure in the @@ -778,7 +778,8 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> { { let (value, _map) = self.tcx.replace_late_bound_regions(value, |br| { debug!(?br); - let liberated_region = self.tcx.mk_re_free(all_outlive_scope.to_def_id(), br.kind); + let liberated_region = + ty::Region::new_free(self.tcx, all_outlive_scope.to_def_id(), br.kind); let region_vid = { let name = match br.kind.get_name() { Some(name) => name, @@ -889,7 +890,7 @@ impl<'tcx> UniversalRegionIndices<'tcx> { where T: TypeFoldable<TyCtxt<'tcx>>, { - tcx.fold_regions(value, |region, _| tcx.mk_re_var(self.to_region_vid(region))) + tcx.fold_regions(value, |region, _| ty::Region::new_var(tcx, self.to_region_vid(region))) } } @@ -929,7 +930,7 @@ fn for_each_late_bound_region_in_item<'tcx>( for bound_var in tcx.late_bound_vars(tcx.hir().local_def_id_to_hir_id(mir_def_id)) { let ty::BoundVariableKind::Region(bound_region) = bound_var else { continue; }; - let liberated_region = tcx.mk_re_free(mir_def_id.to_def_id(), bound_region); + let liberated_region = ty::Region::new_free(tcx, mir_def_id.to_def_id(), bound_region); f(liberated_region); } } diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md index c5222982aa7..26dccf309e1 100644 --- a/compiler/rustc_codegen_cranelift/Readme.md +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -42,6 +42,32 @@ This will build your project with rustc_codegen_cranelift instead of the usual L For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). +## Building and testing with changes in rustc code + +This is useful when changing code in `rustc_codegen_cranelift` as part of changing [main Rust repository](https://github.com/rust-lang/rust/). +This can happen, for example, when you are implementing a new compiler intrinsic. + +Instruction below uses `$RustCheckoutDir` as substitute for any folder where you cloned Rust repository. + +You need to do this steps to successfully compile and use the cranelift backend with your changes in rustc code: + +1. `cd $RustCheckoutDir` +2. Run `python x.py setup` and choose option for compiler (`b`). +3. Build compiler and necessary tools: `python x.py build --stage=2 compiler library/std src/tools/rustdoc src/tools/rustfmt` + * (Optional) You can also build cargo by adding `src/tools/cargo` to previous command. +4. Copy exectutable files from `./build/host/stage2-tools/<your hostname triple>/release` +to `./build/host/stage2/bin/`. Note that you would need to do this every time you rebuilt `rust` repository. +5. Copy cargo from another toolchain: `cp $(rustup which cargo) .build/<your hostname triple>/stage2/bin/cargo` + * Another option is to build it at step 3 and copy with other executables at step 4. +6. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`. +7. (Windows only) compile y.rs: `rustc +stage2 -O y.rs`. +8. You need to prefix every `./y.rs` (or `y` if you built `y.rs`) command by `rustup run stage2` to make cg_clif use your local changes in rustc. + + * `rustup run stage2 ./y.rs prepare` + * `rustup run stage2 ./y.rs build` + * (Optional) run tests: `rustup run stage2 ./y.rs test` +9. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`. + ## Configuration See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 70cb6dfd66f..67fd6d793e0 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -361,7 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { self.instance.subst_mir_and_normalize_erasing_regions( self.tcx, ty::ParamEnv::reveal_all(), - ty::EarlyBinder::new(value), + ty::EarlyBinder::bind(value), ) } diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs index ac04b61a306..bad87db4732 100644 --- a/compiler/rustc_codegen_gcc/src/common.rs +++ b/compiler/rustc_codegen_gcc/src/common.rs @@ -1,17 +1,15 @@ use gccjit::LValue; use gccjit::{RValue, Type, ToRValue}; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{ BaseTypeMethods, ConstMethods, - DerivedTypeMethods, MiscMethods, StaticMethods, }; use rustc_middle::mir::Mutability; -use rustc_middle::ty::layout::{TyAndLayout, LayoutOf}; +use rustc_middle::ty::layout::{LayoutOf}; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_target::abi::{self, HasDataLayout, Pointer, Size}; +use rustc_target::abi::{self, HasDataLayout, Pointer}; use crate::consts::const_alloc_to_gcc; use crate::context::CodegenCx; @@ -240,28 +238,26 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> { const_alloc_to_gcc(self, alloc) } - fn from_const_alloc(&self, layout: TyAndLayout<'tcx>, alloc: ConstAllocation<'tcx>, offset: Size) -> PlaceRef<'tcx, RValue<'gcc>> { - assert_eq!(alloc.inner().align, layout.align.abi); - let ty = self.type_ptr_to(layout.gcc_type(self)); - let value = - if layout.size == Size::ZERO { - let value = self.const_usize(alloc.inner().align.bytes()); - self.const_bitcast(value, ty) - } - else { - let init = const_alloc_to_gcc(self, alloc); - let base_addr = self.static_addr_of(init, alloc.inner().align, None); - - let array = self.const_bitcast(base_addr, self.type_i8p()); - let value = self.context.new_array_access(None, array, self.const_usize(offset.bytes())).get_address(None); - self.const_bitcast(value, ty) - }; - PlaceRef::new_sized(value, layout) - } - fn const_ptrcast(&self, val: RValue<'gcc>, ty: Type<'gcc>) -> RValue<'gcc> { self.context.new_cast(None, val, ty) } + + fn const_bitcast(&self, value: RValue<'gcc>, typ: Type<'gcc>) -> RValue<'gcc> { + if value.get_type() == self.bool_type.make_pointer() { + if let Some(pointee) = typ.get_pointee() { + if pointee.dyncast_vector().is_some() { + panic!() + } + } + } + // NOTE: since bitcast makes a value non-constant, don't bitcast if not necessary as some + // SIMD builtins require a constant value. + self.bitcast_if_needed(value, typ) + } + + fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { + self.context.new_array_access(None, base_addr, self.const_usize(offset.bytes())).get_address(None) + } } pub trait SignType<'gcc, 'tcx> { diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs index 792ab8f890d..873f652e6f1 100644 --- a/compiler/rustc_codegen_gcc/src/consts.rs +++ b/compiler/rustc_codegen_gcc/src/consts.rs @@ -1,6 +1,6 @@ #[cfg(feature = "master")] use gccjit::FnAttribute; -use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue, Type}; +use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue}; use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, DerivedTypeMethods, StaticMethods}; use rustc_middle::span_bug; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -16,21 +16,6 @@ use crate::context::CodegenCx; use crate::errors::InvalidMinimumAlignment; use crate::type_of::LayoutGccExt; -impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { - pub fn const_bitcast(&self, value: RValue<'gcc>, typ: Type<'gcc>) -> RValue<'gcc> { - if value.get_type() == self.bool_type.make_pointer() { - if let Some(pointee) = typ.get_pointee() { - if pointee.dyncast_vector().is_some() { - panic!() - } - } - } - // NOTE: since bitcast makes a value non-constant, don't bitcast if not necessary as some - // SIMD builtins require a constant value. - self.bitcast_if_needed(value, typ) - } -} - fn set_global_alignment<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, gv: LValue<'gcc>, mut align: Align) { // The target may require greater alignment for globals than the type does. // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 9127fba388b..a3910fef954 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -8,16 +8,15 @@ use crate::type_of::LayoutLlvmExt; use crate::value::Value; use rustc_ast::Mutability; -use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher}; use rustc_hir::def_id::DefId; use rustc_middle::bug; use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; -use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; -use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer}; use rustc_target::spec::Target; use libc::{c_char, c_uint}; @@ -307,38 +306,24 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { const_alloc_to_llvm(self, alloc) } - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, &'ll Value> { - let alloc_align = alloc.inner().align; - assert_eq!(alloc_align, layout.align.abi); - let llty = self.type_ptr_to(layout.llvm_type(self)); - let llval = if layout.size == Size::ZERO { - let llval = self.const_usize(alloc_align.bytes()); - unsafe { llvm::LLVMConstIntToPtr(llval, llty) } - } else { - let init = const_alloc_to_llvm(self, alloc); - let base_addr = self.static_addr_of(init, alloc_align, None); - - let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( - self.type_i8(), - self.const_bitcast(base_addr, self.type_i8p()), - &self.const_usize(offset.bytes()), - 1, - ) - }; - self.const_bitcast(llval, llty) - }; - PlaceRef::new_sized(llval, layout) - } - fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { consts::ptrcast(val, ty) } + + fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + self.const_bitcast(val, ty) + } + + fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { + unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p()), + &self.const_usize(offset.bytes()), + 1, + ) + } + } } /// Get the [LLVM type][Type] of a [`Value`]. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 6cb9e163b40..64961baf272 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -93,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( let callee = cx.tcx.subst_and_normalize_erasing_regions( instance.substs, ty::ParamEnv::reveal_all(), - ty::EarlyBinder::new(callee), + ty::EarlyBinder::bind(callee), ); let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); cx.dbg_scope_fn(callee, callee_fn_abi, None) diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 835074806e9..22c1f05974d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -234,7 +234,6 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> | PlaceContext::NonMutatingUse( NonMutatingUseContext::Inspect | NonMutatingUseContext::SharedBorrow - | NonMutatingUseContext::UniqueBorrow | NonMutatingUseContext::ShallowBorrow | NonMutatingUseContext::AddressOf | NonMutatingUseContext::Projection, diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 51393a5d774..00414003202 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -111,7 +111,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.instance.subst_mir_and_normalize_erasing_regions( self.cx.tcx(), ty::ParamEnv::reveal_all(), - ty::EarlyBinder::new(value), + ty::EarlyBinder::bind(value), ) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 2301c3ef13e..4000c9540ce 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -8,10 +8,10 @@ use crate::traits::*; use crate::MemFlags; use rustc_middle::mir; -use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar}; +use rustc_middle::mir::interpret::{alloc_range, ConstValue, Pointer, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::Ty; -use rustc_target::abi::{Abi, Align, Size}; +use rustc_target::abi::{self, Abi, Align, Size}; use std::fmt; @@ -115,13 +115,82 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { OperandValue::Pair(a_llval, b_llval) } ConstValue::ByRef { alloc, offset } => { - return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); + return Self::from_const_alloc(bx, layout, alloc, offset); } }; OperandRef { val, layout } } + fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>( + bx: &mut Bx, + layout: TyAndLayout<'tcx>, + alloc: rustc_middle::mir::interpret::ConstAllocation<'tcx>, + offset: Size, + ) -> Self { + let alloc_align = alloc.inner().align; + assert_eq!(alloc_align, layout.align.abi); + let ty = bx.type_ptr_to(bx.cx().backend_type(layout)); + + let read_scalar = |start, size, s: abi::Scalar, ty| { + let val = alloc + .0 + .read_scalar( + bx, + alloc_range(start, size), + /*read_provenance*/ matches!(s.primitive(), abi::Pointer(_)), + ) + .unwrap(); + bx.scalar_to_backend(val, s, ty) + }; + + // It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point. + // However, `MaybeUninit<u64>` is considered a `Scalar` as far as its layout is concerned -- + // and yet cannot be represented by an interpreter `Scalar`, since we have to handle the + // case where some of the bytes are initialized and others are not. So, we need an extra + // check that walks over the type of `mplace` to make sure it is truly correct to treat this + // like a `Scalar` (or `ScalarPair`). + match layout.abi { + Abi::Scalar(s @ abi::Scalar::Initialized { .. }) => { + let size = s.size(bx); + assert_eq!(size, layout.size, "abi::Scalar size does not match layout size"); + let val = read_scalar(Size::ZERO, size, s, ty); + OperandRef { val: OperandValue::Immediate(val), layout } + } + Abi::ScalarPair( + a @ abi::Scalar::Initialized { .. }, + b @ abi::Scalar::Initialized { .. }, + ) => { + let (a_size, b_size) = (a.size(bx), b.size(bx)); + let b_offset = a_size.align_to(b.align(bx).abi); + assert!(b_offset.bytes() > 0); + let a_val = read_scalar( + Size::ZERO, + a_size, + a, + bx.scalar_pair_element_backend_type(layout, 0, true), + ); + let b_val = read_scalar( + b_offset, + b_size, + b, + bx.scalar_pair_element_backend_type(layout, 1, true), + ); + OperandRef { val: OperandValue::Pair(a_val, b_val), layout } + } + _ if layout.is_zst() => OperandRef::new_zst(bx, layout), + _ => { + // Neither a scalar nor scalar pair. Load from a place + let init = bx.const_data_from_alloc(alloc); + let base_addr = bx.static_addr_of(init, alloc_align, None); + + let llval = bx.const_ptr_byte_offset(base_addr, offset); + let llval = bx.const_bitcast(llval, ty); + bx.load_operand(PlaceRef::new_sized(llval, layout)) + } + } + } + /// Asserts that this operand refers to a scalar and returns /// a reference to its value. pub fn immediate(self) -> V { diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 6e7065713b8..0255b660380 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -668,11 +668,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::NullaryOp(ref null_op, ty) => { let ty = self.monomorphize(ty); - assert!(bx.cx().type_is_sized(ty)); let layout = bx.cx().layout_of(ty); let val = match null_op { - mir::NullOp::SizeOf => layout.size.bytes(), - mir::NullOp::AlignOf => layout.align.abi.bytes(), + mir::NullOp::SizeOf => { + assert!(bx.cx().type_is_sized(ty)); + layout.size.bytes() + } + mir::NullOp::AlignOf => { + assert!(bx.cx().type_is_sized(ty)); + layout.align.abi.bytes() + } mir::NullOp::OffsetOf(fields) => { layout.offset_of_subfield(bx.cx(), fields.iter().map(|f| f.index())).bytes() } diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index 61906302779..dc2fc396480 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -1,8 +1,6 @@ use super::BackendTypes; -use crate::mir::place::PlaceRef; use rustc_middle::mir::interpret::{ConstAllocation, Scalar}; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_target::abi::{self, Size}; +use rustc_target::abi; pub trait ConstMethods<'tcx>: BackendTypes { // Constant constructors @@ -30,12 +28,8 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value; fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; - fn from_const_alloc( - &self, - layout: TyAndLayout<'tcx>, - alloc: ConstAllocation<'tcx>, - offset: Size, - ) -> PlaceRef<'tcx, Self::Value>; fn const_ptrcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; + fn const_bitcast(&self, val: Self::Value, ty: Self::Type) -> Self::Value; + fn const_ptr_byte_offset(&self, val: Self::Value, offset: abi::Size) -> Self::Value; } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 9195ae163bc..91ccdef7215 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -497,7 +497,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .try_subst_mir_and_normalize_erasing_regions( *self.tcx, self.param_env, - ty::EarlyBinder::new(value), + ty::EarlyBinder::bind(value), ) .map_err(|_| err_inval!(TooGeneric)) } diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs index 138bc3eb74a..57d939747aa 100644 --- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs @@ -412,9 +412,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { BorrowKind::Shallow => { PlaceContext::NonMutatingUse(NonMutatingUseContext::ShallowBorrow) } - BorrowKind::Unique => { - PlaceContext::NonMutatingUse(NonMutatingUseContext::UniqueBorrow) - } + BorrowKind::Unique => PlaceContext::MutatingUse(MutatingUseContext::Borrow), BorrowKind::Mut { .. } => { PlaceContext::MutatingUse(MutatingUseContext::Borrow) } diff --git a/compiler/rustc_hir_analysis/src/astconv/mod.rs b/compiler/rustc_hir_analysis/src/astconv/mod.rs index 284ae0e1e58..3d78ea9aa9b 100644 --- a/compiler/rustc_hir_analysis/src/astconv/mod.rs +++ b/compiler/rustc_hir_analysis/src/astconv/mod.rs @@ -239,7 +239,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { var: ty::BoundVar::from_u32(index), kind: ty::BrNamed(def_id, name), }; - tcx.mk_re_late_bound(debruijn, br) + ty::Region::new_late_bound(tcx, debruijn, br) } Some(rbv::ResolvedArg::EarlyBound(def_id)) => { @@ -247,12 +247,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { let item_def_id = tcx.hir().ty_param_owner(def_id.expect_local()); let generics = tcx.generics_of(item_def_id); let index = generics.param_def_id_to_index[&def_id]; - tcx.mk_re_early_bound(ty::EarlyBoundRegion { def_id, index, name }) + ty::Region::new_early_bound(tcx, ty::EarlyBoundRegion { def_id, index, name }) } Some(rbv::ResolvedArg::Free(scope, id)) => { let name = lifetime_name(id.expect_local()); - tcx.mk_re_free(scope, ty::BrNamed(id, name)) + ty::Region::new_free(tcx, scope, ty::BrNamed(id, name)) // (*) -- not late-bound, won't change } @@ -269,7 +269,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { // elision. `resolve_lifetime` should have // reported an error in this case -- but if // not, let's error out. - tcx.mk_re_error_with_message( + ty::Region::new_error_with_message( + tcx, lifetime.ident.span, "unelided lifetime in signature", ) @@ -485,7 +486,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { debug!(?param, "unelided lifetime in signature"); // This indicates an illegal lifetime in a non-assoc-trait position - tcx.mk_re_error_with_message( + ty::Region::new_error_with_message( + tcx, self.span, "unelided lifetime in signature", ) @@ -1219,15 +1221,15 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { let substs = candidate.skip_binder().substs.extend_to(tcx, assoc_item.def_id, |param, _| { let subst = match param.kind { - GenericParamDefKind::Lifetime => tcx - .mk_re_late_bound( - ty::INNERMOST, - ty::BoundRegion { - var: ty::BoundVar::from_usize(num_bound_vars), - kind: ty::BoundRegionKind::BrNamed(param.def_id, param.name), - }, - ) - .into(), + GenericParamDefKind::Lifetime => ty::Region::new_late_bound( + tcx, + ty::INNERMOST, + ty::BoundRegion { + var: ty::BoundVar::from_usize(num_bound_vars), + kind: ty::BoundRegionKind::BrNamed(param.def_id, param.name), + }, + ) + .into(), GenericParamDefKind::Type { .. } => tcx .mk_bound( ty::INNERMOST, @@ -1278,7 +1280,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { // params (and trait ref's late bound params). This logic is very similar to // `Predicate::subst_supertrait`, and it's no coincidence why. let shifted_output = tcx.shift_bound_var_indices(num_bound_vars, output); - let subst_output = ty::EarlyBinder::new(shifted_output).subst(tcx, substs); + let subst_output = ty::EarlyBinder::bind(shifted_output).subst(tcx, substs); let bound_vars = tcx.late_bound_vars(binding.hir_id); ty::Binder::bind_with_vars(subst_output, bound_vars) @@ -1804,7 +1806,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { } else { err.emit() }; - tcx.mk_re_error(e) + ty::Region::new_error(tcx, e) }) } }) diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs index 18b6537a8e9..31b89525f15 100644 --- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs +++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs @@ -472,7 +472,8 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateBound<'_, 'tcx> { fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> { if let ty::ReFree(fr) = *r { - self.tcx.mk_re_free( + ty::Region::new_free( + self.tcx, fr.scope, self.mapping.get(&fr.bound_region).copied().unwrap_or(fr.bound_region), ) @@ -786,23 +787,23 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>( } let Some(ty::ReEarlyBound(e)) = map.get(®ion.into()).map(|r| r.expect_region().kind()) else { - return tcx.mk_re_error_with_message(return_span, "expected ReFree to map to ReEarlyBound") + return ty::Region::new_error_with_message(tcx, return_span, "expected ReFree to map to ReEarlyBound") }; - tcx.mk_re_early_bound(ty::EarlyBoundRegion { + ty::Region::new_early_bound(tcx, ty::EarlyBoundRegion { def_id: e.def_id, name: e.name, index: (e.index as usize - num_trait_substs + num_impl_substs) as u32, }) }); debug!(%ty); - collected_tys.insert(def_id, ty::EarlyBinder::new(ty)); + collected_tys.insert(def_id, ty::EarlyBinder::bind(ty)); } Err(err) => { let reported = tcx.sess.delay_span_bug( return_span, format!("could not fully resolve: {ty} => {err:?}"), ); - collected_tys.insert(def_id, ty::EarlyBinder::new(tcx.ty_error(reported))); + collected_tys.insert(def_id, ty::EarlyBinder::bind(tcx.ty_error(reported))); } } } @@ -1933,7 +1934,8 @@ pub(super) fn check_type_bounds<'tcx>( let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name); let bound_var = ty::BoundVariableKind::Region(kind); bound_vars.push(bound_var); - tcx.mk_re_late_bound( + ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind }, ) diff --git a/compiler/rustc_hir_analysis/src/check/dropck.rs b/compiler/rustc_hir_analysis/src/check/dropck.rs index eeb69c389cd..13d1abe2a65 100644 --- a/compiler/rustc_hir_analysis/src/check/dropck.rs +++ b/compiler/rustc_hir_analysis/src/check/dropck.rs @@ -128,7 +128,7 @@ fn ensure_drop_predicates_are_implied_by_item_defn<'tcx>( // We don't need to normalize this param-env or anything, since we're only // substituting it with free params, so no additional param-env normalization // can occur on top of what has been done in the param_env query itself. - let param_env = ty::EarlyBinder::new(tcx.param_env(adt_def_id)) + let param_env = ty::EarlyBinder::bind(tcx.param_env(adt_def_id)) .subst(tcx, adt_to_impl_substs) .with_constness(tcx.constness(drop_impl_def_id)); @@ -183,7 +183,7 @@ fn ensure_drop_predicates_are_implied_by_item_defn<'tcx>( } RegionResolutionError::SubSupConflict(_, _, _, a, _, b, _) => format!("{b}: {a}"), RegionResolutionError::UpperBoundUniverseConflict(a, _, _, _, b) => { - format!("{b}: {a}", a = tcx.mk_re_var(a)) + format!("{b}: {a}", a = ty::Region::new_var(tcx, a)) } }; guar = Some( diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index fbd4a577d68..1f18017f00b 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -145,11 +145,13 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { ]); let mk_va_list_ty = |mutbl| { tcx.lang_items().va_list().map(|did| { - let region = tcx.mk_re_late_bound( + let region = ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) }, ); - let env_region = tcx.mk_re_late_bound( + let env_region = ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BrEnv }, ); @@ -393,7 +395,12 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) }; ( 1, - vec![tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0))], + vec![ + tcx.mk_imm_ref( + ty::Region::new_late_bound(tcx, ty::INNERMOST, br), + param(0), + ), + ], tcx.mk_projection(discriminant_def_id, tcx.mk_substs(&[param(0).into()])), ) } @@ -443,7 +450,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { sym::raw_eq => { let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(None) }; - let param_ty = tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0)); + let param_ty = + tcx.mk_imm_ref(ty::Region::new_late_bound(tcx, ty::INNERMOST, br), param(0)); (1, vec![param_ty; 2], tcx.types.bool) } diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs index 4c513c4d8cc..fff417fcb29 100644 --- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs +++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs @@ -556,11 +556,14 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>( // Same for the region. In our example, 'a corresponds // to the 'me parameter. let region_param = gat_generics.param_at(*region_a_idx, tcx); - let region_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion { - def_id: region_param.def_id, - index: region_param.index, - name: region_param.name, - }); + let region_param = ty::Region::new_early_bound( + tcx, + ty::EarlyBoundRegion { + def_id: region_param.def_id, + index: region_param.index, + name: region_param.name, + }, + ); // The predicate we expect to see. (In our example, // `Self: 'me`.) let clause = ty::PredicateKind::Clause(ty::Clause::TypeOutlives( @@ -593,18 +596,24 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>( debug!("required clause: {region_a} must outlive {region_b}"); // Translate into the generic parameters of the GAT. let region_a_param = gat_generics.param_at(*region_a_idx, tcx); - let region_a_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion { - def_id: region_a_param.def_id, - index: region_a_param.index, - name: region_a_param.name, - }); + let region_a_param = ty::Region::new_early_bound( + tcx, + ty::EarlyBoundRegion { + def_id: region_a_param.def_id, + index: region_a_param.index, + name: region_a_param.name, + }, + ); // Same for the region. let region_b_param = gat_generics.param_at(*region_b_idx, tcx); - let region_b_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion { - def_id: region_b_param.def_id, - index: region_b_param.index, - name: region_b_param.name, - }); + let region_b_param = ty::Region::new_early_bound( + tcx, + ty::EarlyBoundRegion { + def_id: region_b_param.def_id, + index: region_b_param.index, + name: region_b_param.name, + }, + ); // The predicate we expect to see. let clause = ty::PredicateKind::Clause(ty::Clause::RegionOutlives( ty::OutlivesPredicate(region_a_param, region_b_param), @@ -1398,7 +1407,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id } let mut param_count = CountParams::default(); let has_region = pred.visit_with(&mut param_count).is_break(); - let substituted_pred = ty::EarlyBinder::new(pred).subst(tcx, substs); + let substituted_pred = ty::EarlyBinder::bind(pred).subst(tcx, substs); // Don't check non-defaulted params, dependent defaults (including lifetimes) // or preds with multiple params. if substituted_pred.has_non_region_param() || param_count.params.len() > 1 || has_region diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index 75d99fee152..2f7d465839c 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -440,7 +440,7 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> { self.tcx.replace_late_bound_regions_uncached( poly_trait_ref, |_| { - self.tcx.mk_re_early_bound(ty::EarlyBoundRegion { + ty::Region::new_early_bound(self.tcx, ty::EarlyBoundRegion { def_id: item_def_id, index: 0, name: Symbol::intern(<_name), @@ -1124,7 +1124,7 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<ty::PolyFnSig< bug!("unexpected sort of node in fn_sig(): {:?}", x); } }; - ty::EarlyBinder::new(output) + ty::EarlyBinder::bind(output) } fn infer_return_ty_for_fn_sig<'tcx>( @@ -1312,7 +1312,7 @@ fn impl_trait_ref( check_impl_constness(tcx, impl_.constness, ast_trait_ref), ) }) - .map(ty::EarlyBinder::new) + .map(ty::EarlyBinder::bind) } fn check_impl_constness( diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs index 03c8c59402e..b92d561fb86 100644 --- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs +++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs @@ -86,7 +86,7 @@ pub(super) fn explicit_item_bounds( Some(ty::ImplTraitInTraitData::Trait { opaque_def_id, .. }) => { let item = tcx.hir().get_by_def_id(opaque_def_id.expect_local()).expect_item(); let opaque_ty = item.expect_opaque_ty(); - return ty::EarlyBinder::new(opaque_type_bounds( + return ty::EarlyBinder::bind(opaque_type_bounds( tcx, opaque_def_id.expect_local(), opaque_ty.bounds, @@ -124,7 +124,7 @@ pub(super) fn explicit_item_bounds( } _ => bug!("item_bounds called on {:?}", def_id), }; - ty::EarlyBinder::new(bounds) + ty::EarlyBinder::bind(bounds) } pub(super) fn item_bounds( diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs index e5b5dae551e..70d950eddd8 100644 --- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs @@ -306,11 +306,14 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen let Some(dup_index) = generics.param_def_id_to_index(tcx, dup_def) else { bug!() }; - let dup_region = tcx.mk_re_early_bound(ty::EarlyBoundRegion { - def_id: dup_def, - index: dup_index, - name: duplicate.name.ident().name, - }); + let dup_region = ty::Region::new_early_bound( + tcx, + ty::EarlyBoundRegion { + def_id: dup_def, + index: dup_index, + name: duplicate.name.ident().name, + }, + ); predicates.push(( ty::Binder::dummy(ty::PredicateKind::Clause(ty::Clause::RegionOutlives( ty::OutlivesPredicate(orig_region, dup_region), diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs index 0b57d696585..65ab00fda81 100644 --- a/compiler/rustc_hir_analysis/src/collect/type_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs @@ -323,7 +323,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Ty return map[&assoc_item.trait_item_def_id.unwrap()]; } Err(_) => { - return ty::EarlyBinder::new(tcx.ty_error_with_message( + return ty::EarlyBinder::bind(tcx.ty_error_with_message( DUMMY_SP, "Could not collect return position impl trait in trait tys", )); @@ -497,7 +497,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Ty bug!("unexpected sort of node in type_of(): {:?}", x); } }; - ty::EarlyBinder::new(output) + ty::EarlyBinder::bind(output) } fn infer_placeholder_type<'a>( diff --git a/compiler/rustc_hir_analysis/src/outlives/explicit.rs b/compiler/rustc_hir_analysis/src/outlives/explicit.rs index d7f11d3fd9d..7ce48fe1c01 100644 --- a/compiler/rustc_hir_analysis/src/outlives/explicit.rs +++ b/compiler/rustc_hir_analysis/src/outlives/explicit.rs @@ -68,7 +68,7 @@ impl<'tcx> ExplicitPredicatesMap<'tcx> { } } - ty::EarlyBinder::new(required_predicates) + ty::EarlyBinder::bind(required_predicates) }) } } diff --git a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs index ced994aa636..71dca918fc5 100644 --- a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs +++ b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs @@ -74,7 +74,7 @@ pub(super) fn infer_predicates( if item_required_predicates.len() > item_predicates_len { predicates_added = true; global_inferred_outlives - .insert(item_did.to_def_id(), ty::EarlyBinder::new(item_required_predicates)); + .insert(item_did.to_def_id(), ty::EarlyBinder::bind(item_required_predicates)); } } diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs index 655ab94eb48..4389ad6ef26 100644 --- a/compiler/rustc_hir_typeck/src/callee.rs +++ b/compiler/rustc_hir_typeck/src/callee.rs @@ -420,20 +420,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { .steal_diagnostic(segment.ident.span, StashKey::CallIntoMethod) { // Try suggesting `foo(a)` -> `a.foo()` if possible. - if let Some(ty) = - self.suggest_call_as_method( - &mut diag, - segment, - arg_exprs, - call_expr, - expected - ) - { - diag.emit(); - return ty; - } else { - diag.emit(); - } + self.suggest_call_as_method( + &mut diag, + segment, + arg_exprs, + call_expr, + expected + ); + diag.emit(); } let err = self.report_invalid_callee(call_expr, callee_expr, callee_ty, arg_exprs); @@ -496,9 +490,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { arg_exprs: &'tcx [hir::Expr<'tcx>], call_expr: &'tcx hir::Expr<'tcx>, expected: Expectation<'tcx>, - ) -> Option<Ty<'tcx>> { + ) { if let [callee_expr, rest @ ..] = arg_exprs { - let callee_ty = self.typeck_results.borrow().expr_ty_adjusted_opt(callee_expr)?; + let Some(callee_ty) = self.typeck_results.borrow().expr_ty_adjusted_opt(callee_expr) else { + return; + }; // First, do a probe with `IsSuggestion(true)` to avoid emitting // any strange errors. If it's successful, then we'll do a true @@ -513,7 +509,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ProbeScope::AllTraits, expected.only_has_type(self), ) else { - return None; + return; }; let pick = self.confirm_method( @@ -525,7 +521,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { segment, ); if pick.illegal_sized_bound.is_some() { - return None; + return; } let up_to_rcvr_span = segment.ident.span.until(callee_expr.span); @@ -567,22 +563,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { sugg, Applicability::MaybeIncorrect, ); - - // Let's check the method fully now - let return_ty = self.check_method_argument_types( - segment.ident.span, - call_expr, - Ok(pick.callee), - rest, - TupleArgumentsFlag::DontTupleArguments, - expected, - ); - - return Some(return_ty); } } - - None } fn report_invalid_callee( diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index 08c4082e885..905781ec8f5 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -62,6 +62,7 @@ use rustc_span::{self, BytePos, DesugaringKind, Span}; use rustc_target::spec::abi::Abi; use rustc_trait_selection::infer::InferCtxtExt as _; use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _; +use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use rustc_trait_selection::traits::{ self, NormalizeExt, ObligationCause, ObligationCauseCode, ObligationCtxt, }; @@ -144,12 +145,28 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { debug!("unify(a: {:?}, b: {:?}, use_lub: {})", a, b, self.use_lub); self.commit_if_ok(|_| { let at = self.at(&self.cause, self.fcx.param_env); - if self.use_lub { + + let res = if self.use_lub { at.lub(DefineOpaqueTypes::Yes, b, a) } else { at.sup(DefineOpaqueTypes::Yes, b, a) .map(|InferOk { value: (), obligations }| InferOk { value: a, obligations }) + }; + + // In the new solver, lazy norm may allow us to shallowly equate + // more types, but we emit possibly impossible-to-satisfy obligations. + // Filter these cases out to make sure our coercion is more accurate. + if self.tcx.trait_solver_next() { + if let Ok(res) = &res { + for obligation in &res.obligations { + if !self.predicate_may_hold(&obligation) { + return Err(TypeError::Mismatch); + } + } + } } + + res }) } @@ -791,6 +808,8 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { G: FnOnce(Ty<'tcx>) -> Vec<Adjustment<'tcx>>, { self.commit_if_ok(|snapshot| { + let outer_universe = self.infcx.universe(); + let result = if let ty::FnPtr(fn_ty_b) = b.kind() && let (hir::Unsafety::Normal, hir::Unsafety::Unsafe) = (fn_ty_a.unsafety(), fn_ty_b.unsafety()) @@ -807,7 +826,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> { // want the coerced type to be the actual supertype of these two, // but for now, we want to just error to ensure we don't lock // ourselves into a specific behavior with NLL. - self.leak_check(false, snapshot)?; + self.leak_check(outer_universe, Some(snapshot))?; result }) diff --git a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs index 427d6f8803c..fb28233bfb1 100644 --- a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs +++ b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs @@ -269,7 +269,7 @@ pub fn resolve_interior<'a, 'tcx>( }, _ => mk_bound_region(ty::BrAnon(None)), }; - let r = fcx.tcx.mk_re_late_bound(current_depth, br); + let r = ty::Region::new_late_bound(fcx.tcx, current_depth, br); r }); captured_tys.insert(ty).then(|| { @@ -295,7 +295,11 @@ pub fn resolve_interior<'a, 'tcx>( let var = ty::BoundVar::from_usize(bound_vars.len()); bound_vars.push(ty::BoundVariableKind::Region(kind)); counter += 1; - fcx.tcx.mk_re_late_bound(ty::INNERMOST, ty::BoundRegion { var, kind }) + ty::Region::new_late_bound( + fcx.tcx, + ty::INNERMOST, + ty::BoundRegion { var, kind }, + ) }, types: &mut |b| bug!("unexpected bound ty in binder: {b:?}"), consts: &mut |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"), diff --git a/compiler/rustc_infer/src/infer/at.rs b/compiler/rustc_infer/src/infer/at.rs index 0c8854e962a..6b2dd0a2b4f 100644 --- a/compiler/rustc_infer/src/infer/at.rs +++ b/compiler/rustc_infer/src/infer/at.rs @@ -70,8 +70,8 @@ impl<'tcx> InferCtxt<'tcx> { tcx: self.tcx, defining_use_anchor: self.defining_use_anchor, considering_regions: self.considering_regions, + skip_leak_check: self.skip_leak_check, inner: self.inner.clone(), - skip_leak_check: self.skip_leak_check.clone(), lexical_region_resolutions: self.lexical_region_resolutions.clone(), selection_cache: self.selection_cache.clone(), evaluation_cache: self.evaluation_cache.clone(), diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs index 427d05c8b4d..bf53a73f398 100644 --- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs +++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs @@ -771,7 +771,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { ) -> ty::Region<'tcx> { let var = self.canonical_var(info, r.into()); let br = ty::BoundRegion { var, kind: ty::BrAnon(None) }; - self.interner().mk_re_late_bound(self.binder_index, br) + ty::Region::new_late_bound(self.interner(), self.binder_index, br) } /// Given a type variable `ty_var` of the given kind, first check diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs index 2abdd5b0aec..c8c318c3f02 100644 --- a/compiler/rustc_infer/src/infer/canonical/mod.rs +++ b/compiler/rustc_infer/src/infer/canonical/mod.rs @@ -141,7 +141,7 @@ impl<'tcx> InferCtxt<'tcx> { CanonicalVarKind::PlaceholderRegion(ty::PlaceholderRegion { universe, bound }) => { let universe_mapped = universe_map(universe); let placeholder_mapped = ty::PlaceholderRegion { universe: universe_mapped, bound }; - self.tcx.mk_re_placeholder(placeholder_mapped).into() + ty::Region::new_placeholder(self.tcx, placeholder_mapped).into() } CanonicalVarKind::Const(ui, ty) => self diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs index 88256c819f4..2cf8d8c702d 100644 --- a/compiler/rustc_infer/src/infer/canonical/query_response.rs +++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs @@ -668,14 +668,15 @@ pub fn make_query_region_constraints<'tcx>( let constraint = match *k { // Swap regions because we are going from sub (<=) to outlives // (>=). - Constraint::VarSubVar(v1, v2) => { - ty::OutlivesPredicate(tcx.mk_re_var(v2).into(), tcx.mk_re_var(v1)) - } + Constraint::VarSubVar(v1, v2) => ty::OutlivesPredicate( + ty::Region::new_var(tcx, v2).into(), + ty::Region::new_var(tcx, v1), + ), Constraint::VarSubReg(v1, r2) => { - ty::OutlivesPredicate(r2.into(), tcx.mk_re_var(v1)) + ty::OutlivesPredicate(r2.into(), ty::Region::new_var(tcx, v1)) } Constraint::RegSubVar(r1, v2) => { - ty::OutlivesPredicate(tcx.mk_re_var(v2).into(), r1) + ty::OutlivesPredicate(ty::Region::new_var(tcx, v2).into(), r1) } Constraint::RegSubReg(r1, r2) => ty::OutlivesPredicate(r2.into(), r1), }; @@ -719,7 +720,7 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for QueryTypeRelatingDelegate<'_, 'tcx> { } fn next_placeholder_region(&mut self, placeholder: ty::PlaceholderRegion) -> ty::Region<'tcx> { - self.infcx.tcx.mk_re_placeholder(placeholder) + ty::Region::new_placeholder(self.infcx.tcx, placeholder) } fn generalize_existential(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx> { diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs index c9c1f0aeaac..0b3bc1ce6b3 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs @@ -79,7 +79,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> { sup_placeholder @ Region(Interned(RePlaceholder(_), _)), _, )) => self.try_report_trait_placeholder_mismatch( - Some(self.tcx().mk_re_var(*vid)), + Some(ty::Region::new_var(self.tcx(), *vid)), cause, Some(*sub_placeholder), Some(*sup_placeholder), @@ -95,7 +95,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> { _, _, )) => self.try_report_trait_placeholder_mismatch( - Some(self.tcx().mk_re_var(*vid)), + Some(ty::Region::new_var(self.tcx(), *vid)), cause, Some(*sub_placeholder), None, @@ -111,7 +111,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> { sup_placeholder @ Region(Interned(RePlaceholder(_), _)), _, )) => self.try_report_trait_placeholder_mismatch( - Some(self.tcx().mk_re_var(*vid)), + Some(ty::Region::new_var(self.tcx(), *vid)), cause, None, Some(*sup_placeholder), @@ -127,7 +127,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> { sup_placeholder @ Region(Interned(RePlaceholder(_), _)), _, )) => self.try_report_trait_placeholder_mismatch( - Some(self.tcx().mk_re_var(*vid)), + Some(ty::Region::new_var(self.tcx(), *vid)), cause, None, Some(*sup_placeholder), @@ -141,7 +141,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> { SubregionOrigin::Subtype(box TypeTrace { cause, values }), sup_placeholder @ Region(Interned(RePlaceholder(_), _)), )) => self.try_report_trait_placeholder_mismatch( - Some(self.tcx().mk_re_var(*vid)), + Some(ty::Region::new_var(self.tcx(), *vid)), cause, None, Some(*sup_placeholder), diff --git a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs index c304cd25c9c..974bc2f1153 100644 --- a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs +++ b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs @@ -82,8 +82,10 @@ impl<'tcx> InferCtxt<'tcx> { let delegate = FnMutDelegate { regions: &mut |br: ty::BoundRegion| { - self.tcx - .mk_re_placeholder(ty::PlaceholderRegion { universe: next_universe, bound: br }) + ty::Region::new_placeholder( + self.tcx, + ty::PlaceholderRegion { universe: next_universe, bound: br }, + ) }, types: &mut |bound_ty: ty::BoundTy| { self.tcx.mk_placeholder(ty::PlaceholderType { @@ -103,13 +105,15 @@ impl<'tcx> InferCtxt<'tcx> { self.tcx.replace_bound_vars_uncached(binder, delegate) } - /// See [RegionConstraintCollector::leak_check][1]. + /// See [RegionConstraintCollector::leak_check][1]. We only check placeholder + /// leaking into `outer_universe`, i.e. placeholders which cannot be named by that + /// universe. /// /// [1]: crate::infer::region_constraints::RegionConstraintCollector::leak_check pub fn leak_check( &self, - overly_polymorphic: bool, - snapshot: &CombinedSnapshot<'tcx>, + outer_universe: ty::UniverseIndex, + only_consider_snapshot: Option<&CombinedSnapshot<'tcx>>, ) -> RelateResult<'tcx, ()> { // If the user gave `-Zno-leak-check`, or we have been // configured to skip the leak check, then skip the leak check @@ -117,15 +121,15 @@ impl<'tcx> InferCtxt<'tcx> { // subtyping errors that it would have caught will now be // caught later on, during region checking. However, we // continue to use it for a transition period. - if self.tcx.sess.opts.unstable_opts.no_leak_check || self.skip_leak_check.get() { + if self.tcx.sess.opts.unstable_opts.no_leak_check || self.skip_leak_check { return Ok(()); } self.inner.borrow_mut().unwrap_region_constraints().leak_check( self.tcx, - overly_polymorphic, + outer_universe, self.universe(), - snapshot, + only_consider_snapshot, ) } } diff --git a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs index 8482ae2aa38..485e34fe2bf 100644 --- a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs +++ b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs @@ -347,7 +347,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> { // name the placeholder, then the placeholder is // larger; otherwise, the only ancestor is `'static`. Err(placeholder) if empty_ui.can_name(placeholder.universe) => { - self.tcx().mk_re_placeholder(placeholder) + ty::Region::new_placeholder(self.tcx(), placeholder) } Err(_) => self.tcx().lifetimes.re_static, }; diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index cd99fc31212..447d4c9f84b 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -251,14 +251,13 @@ pub struct InferCtxt<'tcx> { /// solving is left to borrowck instead. pub considering_regions: bool, - pub inner: RefCell<InferCtxtInner<'tcx>>, - /// If set, this flag causes us to skip the 'leak check' during /// higher-ranked subtyping operations. This flag is a temporary one used /// to manage the removal of the leak-check: for the time being, we still run the - /// leak-check, but we issue warnings. This flag can only be set to true - /// when entering a snapshot. - skip_leak_check: Cell<bool>, + /// leak-check, but we issue warnings. + skip_leak_check: bool, + + pub inner: RefCell<InferCtxtInner<'tcx>>, /// Once region inference is done, the values for each variable. lexical_region_resolutions: RefCell<Option<LexicalRegionResolutions<'tcx>>>, @@ -543,6 +542,7 @@ pub struct InferCtxtBuilder<'tcx> { tcx: TyCtxt<'tcx>, defining_use_anchor: DefiningAnchor, considering_regions: bool, + skip_leak_check: bool, /// Whether we are in coherence mode. intercrate: bool, } @@ -557,6 +557,7 @@ impl<'tcx> TyCtxtInferExt<'tcx> for TyCtxt<'tcx> { tcx: self, defining_use_anchor: DefiningAnchor::Error, considering_regions: true, + skip_leak_check: false, intercrate: false, } } @@ -584,6 +585,11 @@ impl<'tcx> InferCtxtBuilder<'tcx> { self } + pub fn skip_leak_check(mut self, skip_leak_check: bool) -> Self { + self.skip_leak_check = skip_leak_check; + self + } + /// Given a canonical value `C` as a starting point, create an /// inference context that contains each of the bound values /// within instantiated as a fresh variable. The `f` closure is @@ -605,11 +611,18 @@ impl<'tcx> InferCtxtBuilder<'tcx> { } pub fn build(&mut self) -> InferCtxt<'tcx> { - let InferCtxtBuilder { tcx, defining_use_anchor, considering_regions, intercrate } = *self; + let InferCtxtBuilder { + tcx, + defining_use_anchor, + considering_regions, + skip_leak_check, + intercrate, + } = *self; InferCtxt { tcx, defining_use_anchor, considering_regions, + skip_leak_check, inner: RefCell::new(InferCtxtInner::new()), lexical_region_resolutions: RefCell::new(None), selection_cache: Default::default(), @@ -619,7 +632,6 @@ impl<'tcx> InferCtxtBuilder<'tcx> { tainted_by_errors: Cell::new(None), err_count_on_creation: tcx.sess.err_count(), in_snapshot: Cell::new(false), - skip_leak_check: Cell::new(false), universe: Cell::new(ty::UniverseIndex::ROOT), intercrate, } @@ -815,32 +827,9 @@ impl<'tcx> InferCtxt<'tcx> { r } - /// If `should_skip` is true, then execute `f` then unroll any bindings it creates. - #[instrument(skip(self, f), level = "debug")] - pub fn probe_maybe_skip_leak_check<R, F>(&self, should_skip: bool, f: F) -> R - where - F: FnOnce(&CombinedSnapshot<'tcx>) -> R, - { - let snapshot = self.start_snapshot(); - let was_skip_leak_check = self.skip_leak_check.get(); - if should_skip { - self.skip_leak_check.set(true); - } - let r = f(&snapshot); - self.rollback_to("probe", snapshot); - self.skip_leak_check.set(was_skip_leak_check); - r - } - - /// Scan the constraints produced since `snapshot` began and returns: - /// - /// - `None` -- if none of them involves "region outlives" constraints. - /// - `Some(true)` -- if there are `'a: 'b` constraints where `'a` or `'b` is a placeholder. - /// - `Some(false)` -- if there are `'a: 'b` constraints but none involve placeholders. - pub fn region_constraints_added_in_snapshot( - &self, - snapshot: &CombinedSnapshot<'tcx>, - ) -> Option<bool> { + /// Scan the constraints produced since `snapshot` and check whether + /// we added any region constraints. + pub fn region_constraints_added_in_snapshot(&self, snapshot: &CombinedSnapshot<'tcx>) -> bool { self.inner .borrow_mut() .unwrap_region_constraints() @@ -1065,7 +1054,7 @@ impl<'tcx> InferCtxt<'tcx> { ) -> ty::Region<'tcx> { let region_var = self.inner.borrow_mut().unwrap_region_constraints().new_region_var(universe, origin); - self.tcx.mk_re_var(region_var) + ty::Region::new_var(self.tcx, region_var) } /// Return the universe that the region `r` was created in. For diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs index 9d5ec228d82..105a3f08c82 100644 --- a/compiler/rustc_infer/src/infer/opaque_types.rs +++ b/compiler/rustc_infer/src/infer/opaque_types.rs @@ -533,17 +533,29 @@ impl<'tcx> InferCtxt<'tcx> { // these are the same span, but not in cases like `-> (impl // Foo, impl Bar)`. let span = cause.span; - let prev = self.inner.borrow_mut().opaque_types().register( - opaque_type_key, - OpaqueHiddenType { ty: hidden_ty, span }, - origin, - ); - let mut obligations = if let Some(prev) = prev { - self.at(&cause, param_env) - .eq_exp(DefineOpaqueTypes::Yes, a_is_expected, prev, hidden_ty)? - .obligations + let mut obligations = if self.intercrate { + // During intercrate we do not define opaque types but instead always + // force ambiguity unless the hidden type is known to not implement + // our trait. + vec![traits::Obligation::new( + self.tcx, + cause.clone(), + param_env, + ty::PredicateKind::Ambiguous, + )] } else { - Vec::new() + let prev = self.inner.borrow_mut().opaque_types().register( + opaque_type_key, + OpaqueHiddenType { ty: hidden_ty, span }, + origin, + ); + if let Some(prev) = prev { + self.at(&cause, param_env) + .eq_exp(DefineOpaqueTypes::Yes, a_is_expected, prev, hidden_ty)? + .obligations + } else { + Vec::new() + } }; self.add_item_bounds_for_hidden_type( diff --git a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs index 89cfc9ea3d1..dd65f66ccd1 100644 --- a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs +++ b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs @@ -3,7 +3,6 @@ use crate::infer::CombinedSnapshot; use rustc_data_structures::{ fx::FxIndexMap, graph::{scc::Sccs, vec_graph::VecGraph}, - undo_log::UndoLogs, }; use rustc_index::Idx; use rustc_middle::ty::error::TypeError; @@ -13,7 +12,9 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { /// Searches new universes created during `snapshot`, looking for /// placeholders that may "leak" out from the universes they are contained /// in. If any leaking placeholders are found, then an `Err` is returned - /// (typically leading to the snapshot being reversed). + /// (typically leading to the snapshot being reversed). This algorithm + /// only looks at placeholders which cannot be named by `outer_universe`, + /// as this is the universe we're currently checking for a leak. /// /// The leak check *used* to be the only way we had to handle higher-ranked /// obligations. Now that we have integrated universes into the region @@ -55,6 +56,12 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { /// * if they must also be equal to a placeholder P, and U cannot name P, report an error, as that /// indicates `P: R` and `R` is in an incompatible universe /// + /// To improve performance and for the old trait solver caching to be sound, this takes + /// an optional snapshot in which case we only look at region constraints added in that + /// snapshot. If we were to not do that the `leak_check` during evaluation can rely on + /// region constraints added outside of that evaluation. As that is not reflected in the + /// cache key this would be unsound. + /// /// # Historical note /// /// Older variants of the leak check used to report errors for these @@ -62,36 +69,21 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { /// /// * R: P1, even if R cannot name P1, because R = 'static is a valid sol'n /// * R: P1, R: P2, as above + #[instrument(level = "debug", skip(self, tcx, only_consider_snapshot), ret)] pub fn leak_check( &mut self, tcx: TyCtxt<'tcx>, - overly_polymorphic: bool, + outer_universe: ty::UniverseIndex, max_universe: ty::UniverseIndex, - snapshot: &CombinedSnapshot<'tcx>, + only_consider_snapshot: Option<&CombinedSnapshot<'tcx>>, ) -> RelateResult<'tcx, ()> { - debug!( - "leak_check(max_universe={:?}, snapshot.universe={:?}, overly_polymorphic={:?})", - max_universe, snapshot.universe, overly_polymorphic - ); - - assert!(UndoLogs::<super::UndoLog<'_>>::in_snapshot(&self.undo_log)); - - let universe_at_start_of_snapshot = snapshot.universe; - if universe_at_start_of_snapshot == max_universe { + if outer_universe == max_universe { return Ok(()); } - let mini_graph = - &MiniGraph::new(tcx, self.undo_log.region_constraints(), &self.storage.data.verifys); + let mini_graph = &MiniGraph::new(tcx, &self, only_consider_snapshot); - let mut leak_check = LeakCheck::new( - tcx, - universe_at_start_of_snapshot, - max_universe, - overly_polymorphic, - mini_graph, - self, - ); + let mut leak_check = LeakCheck::new(tcx, outer_universe, max_universe, mini_graph, self); leak_check.assign_placeholder_values()?; leak_check.propagate_scc_value()?; Ok(()) @@ -100,9 +92,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { struct LeakCheck<'me, 'tcx> { tcx: TyCtxt<'tcx>, - universe_at_start_of_snapshot: ty::UniverseIndex, - /// Only used when reporting region errors. - overly_polymorphic: bool, + outer_universe: ty::UniverseIndex, mini_graph: &'me MiniGraph<'tcx>, rcc: &'me RegionConstraintCollector<'me, 'tcx>, @@ -130,17 +120,15 @@ struct LeakCheck<'me, 'tcx> { impl<'me, 'tcx> LeakCheck<'me, 'tcx> { fn new( tcx: TyCtxt<'tcx>, - universe_at_start_of_snapshot: ty::UniverseIndex, + outer_universe: ty::UniverseIndex, max_universe: ty::UniverseIndex, - overly_polymorphic: bool, mini_graph: &'me MiniGraph<'tcx>, rcc: &'me RegionConstraintCollector<'me, 'tcx>, ) -> Self { let dummy_scc_universe = SccUniverse { universe: max_universe, region: None }; Self { tcx, - universe_at_start_of_snapshot, - overly_polymorphic, + outer_universe, mini_graph, rcc, scc_placeholders: IndexVec::from_elem_n(None, mini_graph.sccs.num_sccs()), @@ -165,7 +153,7 @@ impl<'me, 'tcx> LeakCheck<'me, 'tcx> { // Detect those SCCs that directly contain a placeholder if let ty::RePlaceholder(placeholder) = **region { - if self.universe_at_start_of_snapshot.cannot_name(placeholder.universe) { + if self.outer_universe.cannot_name(placeholder.universe) { self.assign_scc_value(scc, placeholder)?; } } @@ -280,7 +268,7 @@ impl<'me, 'tcx> LeakCheck<'me, 'tcx> { placeholder1: ty::PlaceholderRegion, placeholder2: ty::PlaceholderRegion, ) -> TypeError<'tcx> { - self.error(placeholder1, self.tcx.mk_re_placeholder(placeholder2)) + self.error(placeholder1, ty::Region::new_placeholder(self.tcx, placeholder2)) } fn error( @@ -289,11 +277,7 @@ impl<'me, 'tcx> LeakCheck<'me, 'tcx> { other_region: ty::Region<'tcx>, ) -> TypeError<'tcx> { debug!("error: placeholder={:?}, other_region={:?}", placeholder, other_region); - if self.overly_polymorphic { - TypeError::RegionsOverlyPolymorphic(placeholder.bound.kind, other_region) - } else { - TypeError::RegionsInsufficientlyPolymorphic(placeholder.bound.kind, other_region) - } + TypeError::RegionsInsufficientlyPolymorphic(placeholder.bound.kind, other_region) } } @@ -379,56 +363,70 @@ struct MiniGraph<'tcx> { } impl<'tcx> MiniGraph<'tcx> { - fn new<'a>( + fn new( tcx: TyCtxt<'tcx>, - undo_log: impl Iterator<Item = &'a UndoLog<'tcx>>, - verifys: &[Verify<'tcx>], - ) -> Self - where - 'tcx: 'a, - { + region_constraints: &RegionConstraintCollector<'_, 'tcx>, + only_consider_snapshot: Option<&CombinedSnapshot<'tcx>>, + ) -> Self { let mut nodes = FxIndexMap::default(); let mut edges = Vec::new(); // Note that if `R2: R1`, we get a callback `r1, r2`, so `target` is first parameter. - Self::iterate_undo_log(tcx, undo_log, verifys, |target, source| { - let source_node = Self::add_node(&mut nodes, source); - let target_node = Self::add_node(&mut nodes, target); - edges.push((source_node, target_node)); - }); + Self::iterate_region_constraints( + tcx, + region_constraints, + only_consider_snapshot, + |target, source| { + let source_node = Self::add_node(&mut nodes, source); + let target_node = Self::add_node(&mut nodes, target); + edges.push((source_node, target_node)); + }, + ); let graph = VecGraph::new(nodes.len(), edges); let sccs = Sccs::new(&graph); Self { nodes, sccs } } /// Invokes `each_edge(R1, R2)` for each edge where `R2: R1` - fn iterate_undo_log<'a>( + fn iterate_region_constraints( tcx: TyCtxt<'tcx>, - undo_log: impl Iterator<Item = &'a UndoLog<'tcx>>, - verifys: &[Verify<'tcx>], + region_constraints: &RegionConstraintCollector<'_, 'tcx>, + only_consider_snapshot: Option<&CombinedSnapshot<'tcx>>, mut each_edge: impl FnMut(ty::Region<'tcx>, ty::Region<'tcx>), - ) where - 'tcx: 'a, - { - for undo_entry in undo_log { - match undo_entry { - &AddConstraint(Constraint::VarSubVar(a, b)) => { - each_edge(tcx.mk_re_var(a), tcx.mk_re_var(b)); - } - &AddConstraint(Constraint::RegSubVar(a, b)) => { - each_edge(a, tcx.mk_re_var(b)); - } - &AddConstraint(Constraint::VarSubReg(a, b)) => { - each_edge(tcx.mk_re_var(a), b); - } - &AddConstraint(Constraint::RegSubReg(a, b)) => { - each_edge(a, b); + ) { + let mut each_constraint = |constraint| match constraint { + &Constraint::VarSubVar(a, b) => { + each_edge(ty::Region::new_var(tcx, a), ty::Region::new_var(tcx, b)); + } + &Constraint::RegSubVar(a, b) => { + each_edge(a, ty::Region::new_var(tcx, b)); + } + &Constraint::VarSubReg(a, b) => { + each_edge(ty::Region::new_var(tcx, a), b); + } + &Constraint::RegSubReg(a, b) => { + each_edge(a, b); + } + }; + + if let Some(snapshot) = only_consider_snapshot { + for undo_entry in + region_constraints.undo_log.region_constraints_in_snapshot(&snapshot.undo_snapshot) + { + match undo_entry { + AddConstraint(constraint) => { + each_constraint(constraint); + } + &AddVerify(i) => span_bug!( + region_constraints.data().verifys[i].origin.span(), + "we never add verifications while doing higher-ranked things", + ), + &AddCombination(..) | &AddVar(..) => {} } - &AddVerify(i) => span_bug!( - verifys[i].origin.span(), - "we never add verifications while doing higher-ranked things", - ), - &AddCombination(..) | &AddVar(..) => {} + } + } else { + for (constraint, _origin) in ®ion_constraints.data().constraints { + each_constraint(constraint) } } } diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs index c7a307b89e4..613da8a0b45 100644 --- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs +++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs @@ -400,7 +400,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { data } - pub(super) fn data(&self) -> &RegionConstraintData<'tcx> { + pub fn data(&self) -> &RegionConstraintData<'tcx> { &self.data } @@ -610,13 +610,13 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { let resolved = ut .probe_value(root_vid) .get_value_ignoring_universes() - .unwrap_or_else(|| tcx.mk_re_var(root_vid)); + .unwrap_or_else(|| ty::Region::new_var(tcx, root_vid)); // Don't resolve a variable to a region that it cannot name. if self.var_universe(vid).can_name(self.universe(resolved)) { resolved } else { - tcx.mk_re_var(vid) + ty::Region::new_var(tcx, vid) } } @@ -637,7 +637,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { ) -> Region<'tcx> { let vars = TwoRegions { a, b }; if let Some(&c) = self.combine_map(t).get(&vars) { - return tcx.mk_re_var(c); + return ty::Region::new_var(tcx, c); } let a_universe = self.universe(a); let b_universe = self.universe(b); @@ -645,7 +645,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { let c = self.new_region_var(c_universe, MiscVariable(origin.span())); self.combine_map(t).insert(vars, c); self.undo_log.push(AddCombination(t, vars)); - let new_r = tcx.mk_re_var(c); + let new_r = ty::Region::new_var(tcx, c); for old_r in [a, b] { match t { Glb => self.make_subregion(origin.clone(), new_r, old_r), @@ -683,15 +683,10 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { } /// See `InferCtxt::region_constraints_added_in_snapshot`. - pub fn region_constraints_added_in_snapshot(&self, mark: &Snapshot<'tcx>) -> Option<bool> { + pub fn region_constraints_added_in_snapshot(&self, mark: &Snapshot<'tcx>) -> bool { self.undo_log .region_constraints_in_snapshot(mark) - .map(|&elt| match elt { - AddConstraint(constraint) => Some(constraint.involves_placeholders()), - _ => None, - }) - .max() - .unwrap_or(None) + .any(|&elt| matches!(elt, AddConstraint(_))) } #[inline] diff --git a/compiler/rustc_infer/src/infer/undo_log.rs b/compiler/rustc_infer/src/infer/undo_log.rs index 955c54e8515..25d06b21ec8 100644 --- a/compiler/rustc_infer/src/infer/undo_log.rs +++ b/compiler/rustc_infer/src/infer/undo_log.rs @@ -138,11 +138,9 @@ impl<'tcx> InferCtxtInner<'tcx> { } if self.undo_log.num_open_snapshots == 1 { - // The root snapshot. It's safe to clear the undo log because - // there's no snapshot further out that we might need to roll back - // to. + // After the root snapshot the undo log should be empty. assert!(snapshot.undo_len == 0); - self.undo_log.logs.clear(); + assert!(self.undo_log.logs.is_empty()); } self.undo_log.num_open_snapshots -= 1; @@ -183,15 +181,6 @@ impl<'tcx> InferCtxtUndoLogs<'tcx> { self.logs[s.undo_len..].iter().any(|log| matches!(log, UndoLog::OpaqueTypes(..))) } - pub(crate) fn region_constraints( - &self, - ) -> impl Iterator<Item = &'_ region_constraints::UndoLog<'tcx>> + Clone { - self.logs.iter().filter_map(|log| match log { - UndoLog::RegionConstraintCollector(log) => Some(log), - _ => None, - }) - } - fn assert_open_snapshot(&self, snapshot: &Snapshot<'tcx>) { // Failures here may indicate a failure to follow a stack discipline. assert!(self.logs.len() >= snapshot.undo_len); diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl index d34a3afcba5..e707ac41a05 100644 --- a/compiler/rustc_lint/messages.ftl +++ b/compiler/rustc_lint/messages.ftl @@ -304,6 +304,14 @@ lint_improper_ctypes_union_layout_help = consider adding a `#[repr(C)]` or `#[re lint_improper_ctypes_union_layout_reason = this union has unspecified layout lint_improper_ctypes_union_non_exhaustive = this union is non-exhaustive +# FIXME: we should ordinalize $valid_up_to when we add support for doing so +lint_invalid_from_utf8_checked = calls to `{$method}` with a invalid literal always return an error + .label = the literal was valid UTF-8 up to the {$valid_up_to} bytes + +# FIXME: we should ordinalize $valid_up_to when we add support for doing so +lint_invalid_from_utf8_unchecked = calls to `{$method}` with a invalid literal are undefined behavior + .label = the literal was valid UTF-8 up to the {$valid_up_to} bytes + lint_lintpass_by_hand = implementing `LintPass` by hand .help = try using `declare_lint_pass!` or `impl_lint_pass!` instead diff --git a/compiler/rustc_lint/src/invalid_from_utf8.rs b/compiler/rustc_lint/src/invalid_from_utf8.rs new file mode 100644 index 00000000000..3291286ad67 --- /dev/null +++ b/compiler/rustc_lint/src/invalid_from_utf8.rs @@ -0,0 +1,118 @@ +use std::str::Utf8Error; + +use rustc_ast::{BorrowKind, LitKind}; +use rustc_hir::{Expr, ExprKind}; +use rustc_span::source_map::Spanned; +use rustc_span::sym; + +use crate::lints::InvalidFromUtf8Diag; +use crate::{LateContext, LateLintPass, LintContext}; + +declare_lint! { + /// The `invalid_from_utf8_unchecked` lint checks for calls to + /// `std::str::from_utf8_unchecked` and `std::str::from_utf8_unchecked_mut` + /// with an invalid UTF-8 literal. + /// + /// ### Example + /// + /// ```rust,compile_fail + /// # #[allow(unused)] + /// unsafe { + /// std::str::from_utf8_unchecked(b"Ru\x82st"); + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Creating such a `str` would result in undefined behavior as per documentation + /// for `std::str::from_utf8_unchecked` and `std::str::from_utf8_unchecked_mut`. + pub INVALID_FROM_UTF8_UNCHECKED, + Deny, + "using a non UTF-8 literal in `std::str::from_utf8_unchecked`" +} + +declare_lint! { + /// The `invalid_from_utf8` lint checks for calls to + /// `std::str::from_utf8` and `std::str::from_utf8_mut` + /// with an invalid UTF-8 literal. + /// + /// ### Example + /// + /// ```rust + /// # #[allow(unused)] + /// std::str::from_utf8(b"Ru\x82st"); + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Trying to create such a `str` would always return an error as per documentation + /// for `std::str::from_utf8` and `std::str::from_utf8_mut`. + pub INVALID_FROM_UTF8, + Warn, + "using a non UTF-8 literal in `std::str::from_utf8`" +} + +declare_lint_pass!(InvalidFromUtf8 => [INVALID_FROM_UTF8_UNCHECKED, INVALID_FROM_UTF8]); + +impl<'tcx> LateLintPass<'tcx> for InvalidFromUtf8 { + fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'tcx>) { + if let ExprKind::Call(path, [arg]) = expr.kind + && let ExprKind::Path(ref qpath) = path.kind + && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() + && let Some(diag_item) = cx.tcx.get_diagnostic_name(def_id) + && [sym::str_from_utf8, sym::str_from_utf8_mut, + sym::str_from_utf8_unchecked, sym::str_from_utf8_unchecked_mut].contains(&diag_item) + { + let lint = |utf8_error: Utf8Error| { + let label = arg.span; + let method = diag_item.as_str().strip_prefix("str_").unwrap(); + let method = format!("std::str::{method}"); + let valid_up_to = utf8_error.valid_up_to(); + let is_unchecked_variant = diag_item.as_str().contains("unchecked"); + + cx.emit_spanned_lint( + if is_unchecked_variant { INVALID_FROM_UTF8_UNCHECKED } else { INVALID_FROM_UTF8 }, + expr.span, + if is_unchecked_variant { + InvalidFromUtf8Diag::Unchecked { method, valid_up_to, label } + } else { + InvalidFromUtf8Diag::Checked { method, valid_up_to, label } + } + ) + }; + + match &arg.kind { + ExprKind::Lit(Spanned { node: lit, .. }) => { + if let LitKind::ByteStr(bytes, _) = &lit + && let Err(utf8_error) = std::str::from_utf8(bytes) + { + lint(utf8_error); + } + }, + ExprKind::AddrOf(BorrowKind::Ref, _, Expr { kind: ExprKind::Array(args), .. }) => { + let elements = args.iter().map(|e|{ + match &e.kind { + ExprKind::Lit(Spanned { node: lit, .. }) => match lit { + LitKind::Byte(b) => Some(*b), + LitKind::Int(b, _) => Some(*b as u8), + _ => None + } + _ => None + } + }).collect::<Option<Vec<_>>>(); + + if let Some(elements) = elements + && let Err(utf8_error) = std::str::from_utf8(&elements) + { + lint(utf8_error); + } + } + _ => {} + } + } + } +} diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs index dfddfe09ab3..c62109b2986 100644 --- a/compiler/rustc_lint/src/lib.rs +++ b/compiler/rustc_lint/src/lib.rs @@ -60,6 +60,7 @@ mod expect; mod for_loops_over_fallibles; pub mod hidden_unicode_codepoints; mod internal; +mod invalid_from_utf8; mod late; mod let_underscore; mod levels; @@ -102,6 +103,7 @@ use enum_intrinsics_non_enums::EnumIntrinsicsNonEnums; use for_loops_over_fallibles::*; use hidden_unicode_codepoints::*; use internal::*; +use invalid_from_utf8::*; use let_underscore::*; use map_unit_fn::*; use methods::*; @@ -207,6 +209,7 @@ late_lint_methods!( HardwiredLints: HardwiredLints, ImproperCTypesDeclarations: ImproperCTypesDeclarations, ImproperCTypesDefinitions: ImproperCTypesDefinitions, + InvalidFromUtf8: InvalidFromUtf8, VariantSizeDifferences: VariantSizeDifferences, BoxPointers: BoxPointers, PathStatements: PathStatements, diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs index 6b4d01551ae..746abebeb37 100644 --- a/compiler/rustc_lint/src/lints.rs +++ b/compiler/rustc_lint/src/lints.rs @@ -699,6 +699,25 @@ pub struct ForgetCopyDiag<'a> { pub label: Span, } +// invalid_from_utf8.rs +#[derive(LintDiagnostic)] +pub enum InvalidFromUtf8Diag { + #[diag(lint_invalid_from_utf8_unchecked)] + Unchecked { + method: String, + valid_up_to: usize, + #[label] + label: Span, + }, + #[diag(lint_invalid_from_utf8_checked)] + Checked { + method: String, + valid_up_to: usize, + #[label] + label: Span, + }, +} + // hidden_unicode_codepoints.rs #[derive(LintDiagnostic)] #[diag(lint_hidden_unicode_codepoints)] diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 06fdc6fe030..09e24e012af 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -858,7 +858,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } else { tcx.arena.alloc_from_iter(lazy.decode((self, tcx))) }; - ty::EarlyBinder::new(&*output) + ty::EarlyBinder::bind(&*output) } fn get_variant( diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index f2a7762f003..c673eb5afc7 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -1730,7 +1730,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { ty::Closure(_, substs) => { let constness = self.tcx.constness(def_id.to_def_id()); self.tables.constness.set_some(def_id.to_def_id().index, constness); - record!(self.tables.fn_sig[def_id.to_def_id()] <- ty::EarlyBinder::new(substs.as_closure().sig())); + record!(self.tables.fn_sig[def_id.to_def_id()] <- ty::EarlyBinder::bind(substs.as_closure().sig())); } _ => bug!("closure that is neither generator nor closure"), diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs index 56171314944..29dae67bfca 100644 --- a/compiler/rustc_middle/src/infer/canonical.rs +++ b/compiler/rustc_middle/src/infer/canonical.rs @@ -415,7 +415,7 @@ impl<'tcx> CanonicalVarValues<'tcx> { var: ty::BoundVar::from_usize(i), kind: ty::BrAnon(None), }; - tcx.mk_re_late_bound(ty::INNERMOST, br).into() + ty::Region::new_late_bound(tcx, ty::INNERMOST, br).into() } CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 34e47de969c..63cdba7f327 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -476,7 +476,7 @@ impl<'tcx> Body<'tcx> { /// Returns the return type; it always return first element from `local_decls` array. #[inline] pub fn bound_return_ty(&self) -> ty::EarlyBinder<Ty<'tcx>> { - ty::EarlyBinder::new(self.local_decls[RETURN_PLACE].ty) + ty::EarlyBinder::bind(self.local_decls[RETURN_PLACE].ty) } /// Gets the location of the terminator for the given block. diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs index f31b343c947..1511e255235 100644 --- a/compiler/rustc_middle/src/mir/mono.rs +++ b/compiler/rustc_middle/src/mir/mono.rs @@ -291,10 +291,12 @@ impl<'tcx> CodegenUnit<'tcx> { self.primary = true; } + /// The order of these items is non-determinstic. pub fn items(&self) -> &FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)> { &self.items } + /// The order of these items is non-determinstic. pub fn items_mut(&mut self) -> &mut FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)> { &mut self.items } diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 53fd2dd23a7..a15c419da7a 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -413,7 +413,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> { ty::ReVar(vid) => { let br = ty::BoundRegion { var: ty::BoundVar::new(vid.index()), kind: ty::BrAnon(None) }; - tcx.mk_re_late_bound(depth, br) + ty::Region::new_late_bound(tcx, depth, br) } _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"), }); diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 6d6d71bc87b..3e474c1d377 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -220,6 +220,11 @@ pub enum BorrowKind { /// immutable, but not aliasable. This solves the problem. For /// simplicity, we don't give users the way to express this /// borrow, it's just used when translating closures. + /// + // FIXME(#112072): This is wrong. Unique borrows are mutable borrows except + // that they do not require their pointee to be marked as a mutable. + // They should still be treated as mutable borrows in every other way, + // e.g. for variance or overlap checking. Unique, /// Data is mutable and not aliasable. diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 942654b3074..8d44e929afd 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -650,8 +650,8 @@ macro_rules! make_mir_visitor { BorrowKind::Shallow => PlaceContext::NonMutatingUse( NonMutatingUseContext::ShallowBorrow ), - BorrowKind::Unique => PlaceContext::NonMutatingUse( - NonMutatingUseContext::UniqueBorrow + BorrowKind::Unique => PlaceContext::MutatingUse( + MutatingUseContext::Borrow ), BorrowKind::Mut { .. } => PlaceContext::MutatingUse(MutatingUseContext::Borrow), @@ -1265,8 +1265,6 @@ pub enum NonMutatingUseContext { SharedBorrow, /// Shallow borrow. ShallowBorrow, - /// Unique borrow. - UniqueBorrow, /// AddressOf for *const pointer. AddressOf, /// PlaceMention statement. @@ -1345,9 +1343,7 @@ impl PlaceContext { matches!( self, PlaceContext::NonMutatingUse( - NonMutatingUseContext::SharedBorrow - | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::UniqueBorrow + NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow ) | PlaceContext::MutatingUse(MutatingUseContext::Borrow) ) } diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 2b99fcad267..e067d2a984f 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -573,7 +573,7 @@ impl<'tcx> AdtDef<'tcx> { /// Due to normalization being eager, this applies even if /// the associated type is behind a pointer (e.g., issue #31299). pub fn sized_constraint(self, tcx: TyCtxt<'tcx>) -> ty::EarlyBinder<&'tcx [Ty<'tcx>]> { - ty::EarlyBinder::new(tcx.adt_sized_constraint(self.did())) + ty::EarlyBinder::bind(tcx.adt_sized_constraint(self.did())) } } diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs index 7fc75674da5..76f52bc34ed 100644 --- a/compiler/rustc_middle/src/ty/codec.rs +++ b/compiler/rustc_middle/src/ty/codec.rs @@ -264,7 +264,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for mir::Place<'tcx> { impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::Region<'tcx> { fn decode(decoder: &mut D) -> Self { - decoder.interner().mk_region_from_kind(Decodable::decode(decoder)) + ty::Region::new_from_kind(decoder.interner(), Decodable::decode(decoder)) } } diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 5fcbc4a9ed4..aecb46556b0 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -254,5 +254,5 @@ pub fn const_param_default(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBind "`const_param_default` expected a generic parameter with a constant" ), }; - ty::EarlyBinder::new(Const::from_anon_const(tcx, default_def_id)) + ty::EarlyBinder::bind(Const::from_anon_const(tcx, default_def_id)) } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 0464336627f..77725f0b3b6 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -115,6 +115,16 @@ impl<'tcx> Interner for TyCtxt<'tcx> { type FreeRegion = ty::FreeRegion; type RegionVid = ty::RegionVid; type PlaceholderRegion = ty::PlaceholderRegion; + + fn ty_and_mut_to_parts( + TypeAndMut { ty, mutbl }: TypeAndMut<'tcx>, + ) -> (Self::Ty, Self::Mutability) { + (ty, mutbl) + } + + fn mutability_is_mut(mutbl: Self::Mutability) -> bool { + mutbl.is_mut() + } } type InternedSet<'tcx, T> = ShardedHashMap<InternedInSet<'tcx, T>, ()>; @@ -713,34 +723,6 @@ impl<'tcx> TyCtxt<'tcx> { self.mk_ty_from_kind(Error(reported)) } - /// Constructs a `RegionKind::ReError` lifetime. - #[track_caller] - pub fn mk_re_error(self, reported: ErrorGuaranteed) -> Region<'tcx> { - self.intern_region(ty::ReError(reported)) - } - - /// Constructs a `RegionKind::ReError` lifetime and registers a `delay_span_bug` to ensure it - /// gets used. - #[track_caller] - pub fn mk_re_error_misc(self) -> Region<'tcx> { - self.mk_re_error_with_message( - DUMMY_SP, - "RegionKind::ReError constructed but no error reported", - ) - } - - /// Constructs a `RegionKind::ReError` lifetime and registers a `delay_span_bug` with the given - /// `msg` to ensure it gets used. - #[track_caller] - pub fn mk_re_error_with_message<S: Into<MultiSpan>>( - self, - span: S, - msg: &'static str, - ) -> Region<'tcx> { - let reported = self.sess.delay_span_bug(span, msg); - self.mk_re_error(reported) - } - /// Like [TyCtxt::ty_error] but for constants, with current `ErrorGuaranteed` #[track_caller] pub fn const_error(self, ty: Ty<'tcx>, reported: ErrorGuaranteed) -> Const<'tcx> { @@ -1519,9 +1501,9 @@ macro_rules! direct_interners { // Functions with a `mk_` prefix are intended for use outside this file and // crate. Functions with an `intern_` prefix are intended for use within this -// file only, and have a corresponding `mk_` function. +// crate only, and have a corresponding `mk_` function. direct_interners! { - region: intern_region(RegionKind<'tcx>): Region -> Region<'tcx>, + region: pub(crate) intern_region(RegionKind<'tcx>): Region -> Region<'tcx>, const_: intern_const(ConstData<'tcx>): Const -> Const<'tcx>, const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>, layout: pub mk_layout(LayoutS): Layout -> Layout<'tcx>, @@ -1996,7 +1978,7 @@ impl<'tcx> TyCtxt<'tcx> { pub fn mk_param_from_def(self, param: &ty::GenericParamDef) -> GenericArg<'tcx> { match param.kind { GenericParamDefKind::Lifetime => { - self.mk_re_early_bound(param.to_early_bound_region_data()).into() + ty::Region::new_early_bound(self, param.to_early_bound_region_data()).into() } GenericParamDefKind::Type { .. } => self.mk_ty_param(param.index, param.name).into(), GenericParamDefKind::Const { .. } => self @@ -2036,65 +2018,6 @@ impl<'tcx> TyCtxt<'tcx> { self.mk_alias(ty::Opaque, self.mk_alias_ty(def_id, substs)) } - #[inline] - pub fn mk_re_early_bound(self, early_bound_region: ty::EarlyBoundRegion) -> Region<'tcx> { - self.intern_region(ty::ReEarlyBound(early_bound_region)) - } - - #[inline] - pub fn mk_re_late_bound( - self, - debruijn: ty::DebruijnIndex, - bound_region: ty::BoundRegion, - ) -> Region<'tcx> { - // Use a pre-interned one when possible. - if let ty::BoundRegion { var, kind: ty::BrAnon(None) } = bound_region - && let Some(inner) = self.lifetimes.re_late_bounds.get(debruijn.as_usize()) - && let Some(re) = inner.get(var.as_usize()).copied() - { - re - } else { - self.intern_region(ty::ReLateBound(debruijn, bound_region)) - } - } - - #[inline] - pub fn mk_re_free(self, scope: DefId, bound_region: ty::BoundRegionKind) -> Region<'tcx> { - self.intern_region(ty::ReFree(ty::FreeRegion { scope, bound_region })) - } - - #[inline] - pub fn mk_re_var(self, v: ty::RegionVid) -> Region<'tcx> { - // Use a pre-interned one when possible. - self.lifetimes - .re_vars - .get(v.as_usize()) - .copied() - .unwrap_or_else(|| self.intern_region(ty::ReVar(v))) - } - - #[inline] - pub fn mk_re_placeholder(self, placeholder: ty::PlaceholderRegion) -> Region<'tcx> { - self.intern_region(ty::RePlaceholder(placeholder)) - } - - // Avoid this in favour of more specific `mk_re_*` methods, where possible, - // to avoid the cost of the `match`. - pub fn mk_region_from_kind(self, kind: ty::RegionKind<'tcx>) -> Region<'tcx> { - match kind { - ty::ReEarlyBound(region) => self.mk_re_early_bound(region), - ty::ReLateBound(debruijn, region) => self.mk_re_late_bound(debruijn, region), - ty::ReFree(ty::FreeRegion { scope, bound_region }) => { - self.mk_re_free(scope, bound_region) - } - ty::ReStatic => self.lifetimes.re_static, - ty::ReVar(vid) => self.mk_re_var(vid), - ty::RePlaceholder(region) => self.mk_re_placeholder(region), - ty::ReErased => self.lifetimes.re_erased, - ty::ReError(reported) => self.mk_re_error(reported), - } - } - pub fn mk_place_field(self, place: Place<'tcx>, f: FieldIdx, ty: Ty<'tcx>) -> Place<'tcx> { self.mk_place_elem(place, PlaceElem::Field(f, ty)) } diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 49ab9b79e96..66293f19eef 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -45,7 +45,6 @@ pub enum TypeError<'tcx> { RegionsDoesNotOutlive(Region<'tcx>, Region<'tcx>), RegionsInsufficientlyPolymorphic(BoundRegionKind, Region<'tcx>), - RegionsOverlyPolymorphic(BoundRegionKind, Region<'tcx>), RegionsPlaceholderMismatch, Sorts(ExpectedFound<Ty<'tcx>>), @@ -74,7 +73,6 @@ impl TypeError<'_> { match self { TypeError::RegionsDoesNotOutlive(_, _) | TypeError::RegionsInsufficientlyPolymorphic(_, _) - | TypeError::RegionsOverlyPolymorphic(_, _) | TypeError::RegionsPlaceholderMismatch => true, _ => false, } @@ -98,11 +96,6 @@ impl<'tcx> TypeError<'tcx> { } } - let br_string = |br: ty::BoundRegionKind| match br { - ty::BrNamed(_, name) => format!(" {}", name), - _ => String::new(), - }; - match self { CyclicTy(_) => "cyclic type of infinite size".into(), CyclicConst(_) => "encountered a self-referencing constant".into(), @@ -144,11 +137,6 @@ impl<'tcx> TypeError<'tcx> { RegionsInsufficientlyPolymorphic(..) => { "one type is more general than the other".into() } - RegionsOverlyPolymorphic(br, _) => format!( - "expected concrete lifetime, found bound lifetime parameter{}", - br_string(br) - ) - .into(), RegionsPlaceholderMismatch => "one type is more general than the other".into(), ArgumentSorts(values, _) | Sorts(values) => { let expected = values.expected.sort_string(tcx); @@ -228,7 +216,6 @@ impl<'tcx> TypeError<'tcx> { | FieldMisMatch(..) | RegionsDoesNotOutlive(..) | RegionsInsufficientlyPolymorphic(..) - | RegionsOverlyPolymorphic(..) | RegionsPlaceholderMismatch | Traits(_) | ProjectionMismatched(_) diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs index 25890eb15cd..149ce29b8d9 100644 --- a/compiler/rustc_middle/src/ty/fold.rs +++ b/compiler/rustc_middle/src/ty/fold.rs @@ -213,7 +213,7 @@ where // debruijn index. Then we adjust it to the // correct depth. assert_eq!(debruijn1, ty::INNERMOST); - self.tcx.mk_re_late_bound(debruijn, br) + ty::Region::new_late_bound(self.tcx, debruijn, br) } else { region } @@ -328,7 +328,7 @@ impl<'tcx> TyCtxt<'tcx> { T: TypeFoldable<TyCtxt<'tcx>>, { self.replace_late_bound_regions_uncached(value, |br| { - self.mk_re_free(all_outlive_scope, br.kind) + ty::Region::new_free(self, all_outlive_scope, br.kind) }) } @@ -341,7 +341,8 @@ impl<'tcx> TyCtxt<'tcx> { value, FnMutDelegate { regions: &mut |r: ty::BoundRegion| { - self.mk_re_late_bound( + ty::Region::new_late_bound( + self, ty::INNERMOST, ty::BoundRegion { var: shift_bv(r.var), kind: r.kind }, ) @@ -383,7 +384,7 @@ impl<'tcx> TyCtxt<'tcx> { .or_insert_with(|| ty::BoundVariableKind::Region(ty::BrAnon(None))) .expect_region(); let br = ty::BoundRegion { var, kind }; - self.tcx.mk_re_late_bound(ty::INNERMOST, br) + ty::Region::new_late_bound(self.tcx, ty::INNERMOST, br) } fn replace_ty(&mut self, bt: ty::BoundTy) -> Ty<'tcx> { let entry = self.map.entry(bt.var); @@ -451,7 +452,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Shifter<'tcx> { match *r { ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => { let debruijn = debruijn.shifted_in(self.amount); - self.tcx.mk_re_late_bound(debruijn, br) + ty::Region::new_late_bound(self.tcx, debruijn, br) } _ => r, } @@ -492,7 +493,7 @@ pub fn shift_region<'tcx>( ) -> ty::Region<'tcx> { match *region { ty::ReLateBound(debruijn, br) if amount > 0 => { - tcx.mk_re_late_bound(debruijn.shifted_in(amount), br) + ty::Region::new_late_bound(tcx, debruijn.shifted_in(amount), br) } _ => region, } diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs index b27ee8cbad0..30f92b91cb7 100644 --- a/compiler/rustc_middle/src/ty/generics.rs +++ b/compiler/rustc_middle/src/ty/generics.rs @@ -100,7 +100,7 @@ impl GenericParamDef { preceding_substs: &[ty::GenericArg<'tcx>], ) -> ty::GenericArg<'tcx> { match &self.kind { - ty::GenericParamDefKind::Lifetime => tcx.mk_re_error_misc().into(), + ty::GenericParamDefKind::Lifetime => ty::Region::new_error_misc(tcx).into(), ty::GenericParamDefKind::Type { .. } => tcx.ty_error_misc().into(), ty::GenericParamDefKind::Const { .. } => { tcx.const_error_misc(tcx.type_of(self.def_id).subst(tcx, preceding_substs)).into() @@ -343,7 +343,7 @@ impl<'tcx> GenericPredicates<'tcx> { substs: SubstsRef<'tcx>, ) -> impl Iterator<Item = (Predicate<'tcx>, Span)> + DoubleEndedIterator + ExactSizeIterator { - EarlyBinder::new(self.predicates).subst_iter_copied(tcx, substs) + EarlyBinder::bind(self.predicates).subst_iter_copied(tcx, substs) } #[instrument(level = "debug", skip(self, tcx))] @@ -358,7 +358,7 @@ impl<'tcx> GenericPredicates<'tcx> { } instantiated .predicates - .extend(self.predicates.iter().map(|(p, _)| EarlyBinder::new(*p).subst(tcx, substs))); + .extend(self.predicates.iter().map(|(p, _)| EarlyBinder::bind(*p).subst(tcx, substs))); instantiated.spans.extend(self.predicates.iter().map(|(_, sp)| *sp)); } diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs index b3cc27e3f78..d48672b2baa 100644 --- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs +++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs @@ -158,7 +158,7 @@ impl<'tcx> InhabitedPredicate<'tcx> { fn subst_opt(self, tcx: TyCtxt<'tcx>, substs: ty::SubstsRef<'tcx>) -> Option<Self> { match self { Self::ConstIsZero(c) => { - let c = ty::EarlyBinder::new(c).subst(tcx, substs); + let c = ty::EarlyBinder::bind(c).subst(tcx, substs); let pred = match c.kind().try_to_target_usize(tcx) { Some(0) => Self::True, Some(1..) => Self::False, @@ -167,7 +167,7 @@ impl<'tcx> InhabitedPredicate<'tcx> { Some(pred) } Self::GenericType(t) => { - Some(ty::EarlyBinder::new(t).subst(tcx, substs).inhabited_predicate(tcx)) + Some(ty::EarlyBinder::bind(t).subst(tcx, substs).inhabited_predicate(tcx)) } Self::And(&[a, b]) => match a.subst_opt(tcx, substs) { None => b.subst_opt(tcx, substs).map(|b| a.and(tcx, b)), diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 4b2c7e8ef31..72341e656e2 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -764,7 +764,7 @@ impl<'tcx> Predicate<'tcx> { let shifted_pred = tcx.shift_bound_var_indices(trait_bound_vars.len(), bound_pred.skip_binder()); // 2) Self: Bar1<'a, '^0.1> -> T: Bar1<'^0.0, '^0.1> - let new = EarlyBinder::new(shifted_pred).subst(tcx, trait_ref.skip_binder().substs); + let new = EarlyBinder::bind(shifted_pred).subst(tcx, trait_ref.skip_binder().substs); // 3) ['x] + ['b] -> ['x, 'b] let bound_vars = tcx.mk_bound_variable_kinds_from_iter(trait_bound_vars.iter().chain(pred_bound_vars)); @@ -1496,7 +1496,7 @@ impl<'tcx> OpaqueHiddenType<'tcx> { /// identified by both a universe, as well as a name residing within that universe. Distinct bound /// regions/types/consts within the same universe simply have an unknown relationship to one /// another. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[derive(HashStable, TyEncodable, TyDecodable)] pub struct Placeholder<T> { pub universe: UniverseIndex, diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs index 1b336b7bfc6..d1ed7be3d2e 100644 --- a/compiler/rustc_middle/src/ty/opaque_types.rs +++ b/compiler/rustc_middle/src/ty/opaque_types.rs @@ -141,7 +141,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> { ) .emit(); - self.interner().mk_re_error(e) + ty::Region::new_error(self.interner(), e) } } } diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index d6c88ea96ca..6156fdf7eac 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -184,7 +184,7 @@ impl<'tcx> RegionHighlightMode<'tcx> { /// Convenience wrapper for `highlighting_region`. pub fn highlighting_region_vid(&mut self, vid: ty::RegionVid, number: usize) { - self.highlighting_region(self.tcx.mk_re_var(vid), number) + self.highlighting_region(ty::Region::new_var(self.tcx, vid), number) } /// Returns `Some(n)` with the number to use for the given region, if any. @@ -685,29 +685,30 @@ pub trait PrettyPrinter<'tcx>: } ty::FnPtr(ref bare_fn) => p!(print(bare_fn)), ty::Infer(infer_ty) => { - let verbose = self.should_print_verbose(); + if self.should_print_verbose() { + p!(write("{:?}", ty.kind())); + return Ok(self); + } + if let ty::TyVar(ty_vid) = infer_ty { if let Some(name) = self.ty_infer_name(ty_vid) { p!(write("{}", name)) } else { - if verbose { - p!(write("{:?}", infer_ty)) - } else { - p!(write("{}", infer_ty)) - } + p!(write("{}", infer_ty)) } } else { - if verbose { p!(write("{:?}", infer_ty)) } else { p!(write("{}", infer_ty)) } + p!(write("{}", infer_ty)) } } ty::Error(_) => p!("{{type error}}"), ty::Param(ref param_ty) => p!(print(param_ty)), ty::Bound(debruijn, bound_ty) => match bound_ty.kind { - ty::BoundTyKind::Anon => debug_bound_var(&mut self, debruijn, bound_ty.var)?, + ty::BoundTyKind::Anon => { + rustc_type_ir::debug_bound_var(&mut self, debruijn, bound_ty.var)? + } ty::BoundTyKind::Param(_, s) => match self.should_print_verbose() { - true if debruijn == ty::INNERMOST => p!(write("^{}", s)), - true => p!(write("^{}_{}", debruijn.index(), s)), - false => p!(write("{}", s)), + true => p!(write("{:?}", ty.kind())), + false => p!(write("{s}")), }, }, ty::Adt(def, substs) => { @@ -740,10 +741,11 @@ pub trait PrettyPrinter<'tcx>: } } ty::Placeholder(placeholder) => match placeholder.bound.kind { - ty::BoundTyKind::Anon => { - debug_placeholder_var(&mut self, placeholder.universe, placeholder.bound.var)?; - } - ty::BoundTyKind::Param(_, name) => p!(write("{}", name)), + ty::BoundTyKind::Anon => p!(write("{placeholder:?}")), + ty::BoundTyKind::Param(_, name) => match self.should_print_verbose() { + true => p!(write("{:?}", ty.kind())), + false => p!(write("{name}")), + }, }, ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => { // We use verbose printing in 'NO_QUERIES' mode, to @@ -1372,11 +1374,9 @@ pub trait PrettyPrinter<'tcx>: } ty::ConstKind::Bound(debruijn, bound_var) => { - debug_bound_var(&mut self, debruijn, bound_var)? + rustc_type_ir::debug_bound_var(&mut self, debruijn, bound_var)? } - ty::ConstKind::Placeholder(placeholder) => { - debug_placeholder_var(&mut self, placeholder.universe, placeholder.bound)?; - }, + ty::ConstKind::Placeholder(placeholder) => p!(write("{placeholder:?}")), // FIXME(generic_const_exprs): // write out some legible representation of an abstract const? ty::ConstKind::Expr(_) => p!("{{const expr}}"), @@ -2303,7 +2303,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> { }; if let ty::ReLateBound(debruijn1, br) = *region { assert_eq!(debruijn1, ty::INNERMOST); - self.tcx.mk_re_late_bound(self.current_index, br) + ty::Region::new_late_bound(self.tcx, self.current_index, br) } else { region } @@ -2415,7 +2415,8 @@ impl<'tcx> FmtPrinter<'_, 'tcx> { if let Some(lt_idx) = lifetime_idx { if lt_idx > binder_level_idx { let kind = ty::BrNamed(CRATE_DEF_ID.to_def_id(), name); - return tcx.mk_re_late_bound( + return ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: br.var, kind }, ); @@ -2430,7 +2431,8 @@ impl<'tcx> FmtPrinter<'_, 'tcx> { if let Some(lt_idx) = lifetime_idx { if lt_idx > binder_level_idx { let kind = ty::BrNamed(def_id, name); - return tcx.mk_re_late_bound( + return ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: br.var, kind }, ); @@ -2443,7 +2445,8 @@ impl<'tcx> FmtPrinter<'_, 'tcx> { if let Some(lt_idx) = lifetime_idx { if lt_idx > binder_level_idx { let kind = br.kind; - return tcx.mk_re_late_bound( + return ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: br.var, kind }, ); @@ -2458,7 +2461,11 @@ impl<'tcx> FmtPrinter<'_, 'tcx> { start_or_continue(&mut self, "for<", ", "); do_continue(&mut self, name); } - tcx.mk_re_late_bound(ty::INNERMOST, ty::BoundRegion { var: br.var, kind }) + ty::Region::new_late_bound( + tcx, + ty::INNERMOST, + ty::BoundRegion { var: br.var, kind }, + ) }; let mut folder = RegionFolder { tcx, @@ -3065,27 +3072,3 @@ pub struct OpaqueFnEntry<'tcx> { fn_trait_ref: Option<ty::PolyTraitRef<'tcx>>, return_ty: Option<ty::Binder<'tcx, Term<'tcx>>>, } - -pub fn debug_bound_var<T: std::fmt::Write>( - fmt: &mut T, - debruijn: ty::DebruijnIndex, - var: ty::BoundVar, -) -> Result<(), std::fmt::Error> { - if debruijn == ty::INNERMOST { - write!(fmt, "^{}", var.index()) - } else { - write!(fmt, "^{}_{}", debruijn.index(), var.index()) - } -} - -pub fn debug_placeholder_var<T: std::fmt::Write>( - fmt: &mut T, - universe: ty::UniverseIndex, - bound: ty::BoundVar, -) -> Result<(), std::fmt::Error> { - if universe == ty::UniverseIndex::ROOT { - write!(fmt, "!{}", bound.index()) - } else { - write!(fmt, "!{}_{}", universe.index(), bound.index()) - } -} diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs index 3bbe6a23b66..8bcae3d9ab7 100644 --- a/compiler/rustc_middle/src/ty/relate.rs +++ b/compiler/rustc_middle/src/ty/relate.rs @@ -589,17 +589,6 @@ pub fn structurally_relate_consts<'tcx, R: TypeRelation<'tcx>>( debug!("{}.structurally_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b); let tcx = relation.tcx(); - // HACK(const_generics): We still need to eagerly evaluate consts when - // relating them because during `normalize_param_env_or_error`, - // we may relate an evaluated constant in a obligation against - // an unnormalized (i.e. unevaluated) const in the param-env. - // FIXME(generic_const_exprs): Once we always lazily unify unevaluated constants - // these `eval` calls can be removed. - if !tcx.features().generic_const_exprs { - a = a.eval(tcx, relation.param_env()); - b = b.eval(tcx, relation.param_env()); - } - if tcx.features().generic_const_exprs { a = tcx.expand_abstract_consts(a); b = tcx.expand_abstract_consts(b); diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 16cb6c91046..a965450d27d 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -88,7 +88,35 @@ impl fmt::Debug for ty::FreeRegion { impl<'tcx> fmt::Debug for ty::FnSig<'tcx> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "({:?}; c_variadic: {})->{:?}", self.inputs(), self.c_variadic, self.output()) + let ty::FnSig { inputs_and_output: _, c_variadic, unsafety, abi } = self; + + write!(f, "{}", unsafety.prefix_str())?; + match abi { + rustc_target::spec::abi::Abi::Rust => (), + abi => write!(f, "extern \"{abi:?}\" ")?, + }; + + write!(f, "fn(")?; + let inputs = self.inputs(); + match inputs.len() { + 0 if *c_variadic => write!(f, "...)")?, + 0 => write!(f, ")")?, + _ => { + for ty in &self.inputs()[0..(self.inputs().len() - 1)] { + write!(f, "{ty:?}, ")?; + } + write!(f, "{:?}", self.inputs().last().unwrap())?; + if *c_variadic { + write!(f, "...")?; + } + write!(f, ")")?; + } + } + + match self.output().kind() { + ty::Tuple(list) if list.is_empty() => Ok(()), + _ => write!(f, " -> {:?}", self.output()), + } } } @@ -216,20 +244,37 @@ impl<'tcx> fmt::Debug for ty::ConstKind<'tcx> { match self { Param(param) => write!(f, "{param:?}"), Infer(var) => write!(f, "{var:?}"), - Bound(debruijn, var) => ty::print::debug_bound_var(f, *debruijn, *var), - Placeholder(placeholder) => { - ty::print::debug_placeholder_var(f, placeholder.universe, placeholder.bound) - } + Bound(debruijn, var) => rustc_type_ir::debug_bound_var(f, *debruijn, *var), + Placeholder(placeholder) => write!(f, "{placeholder:?}"), Unevaluated(uv) => { f.debug_tuple("Unevaluated").field(&uv.substs).field(&uv.def).finish() } Value(valtree) => write!(f, "{valtree:?}"), - Error(_) => write!(f, "[const error]"), + Error(_) => write!(f, "{{const error}}"), Expr(expr) => write!(f, "{expr:?}"), } } } +impl fmt::Debug for ty::BoundTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + ty::BoundTyKind::Anon => write!(f, "{:?}", self.var), + ty::BoundTyKind::Param(_, sym) => write!(f, "{sym:?}"), + } + } +} + +impl<T: fmt::Debug> fmt::Debug for ty::Placeholder<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.universe == ty::UniverseIndex::ROOT { + write!(f, "!{:?}", self.bound) + } else { + write!(f, "!{}_{:?}", self.universe.index(), self.bound) + } + } +} + /////////////////////////////////////////////////////////////////////////// // Atomic structs // @@ -294,6 +339,7 @@ TrivialTypeTraversalAndLiftImpls! { crate::ty::AliasRelationDirection, crate::ty::Placeholder<crate::ty::BoundRegion>, crate::ty::Placeholder<crate::ty::BoundTy>, + crate::ty::Placeholder<ty::BoundVar>, crate::ty::ClosureKind, crate::ty::FreeRegion, crate::ty::InferTy, @@ -310,7 +356,6 @@ TrivialTypeTraversalAndLiftImpls! { interpret::Scalar, rustc_target::abi::Size, ty::BoundVar, - ty::Placeholder<ty::BoundVar>, } TrivialTypeTraversalAndLiftImpls! { diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 82f79a0c088..d19a7bcde79 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -15,14 +15,14 @@ use hir::def::DefKind; use polonius_engine::Atom; use rustc_data_structures::captures::Captures; use rustc_data_structures::intern::Interned; -use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg}; +use rustc_errors::{DiagnosticArgValue, ErrorGuaranteed, IntoDiagnosticArg, MultiSpan}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::LangItem; use rustc_index::Idx; use rustc_macros::HashStable; use rustc_span::symbol::{kw, sym, Symbol}; -use rustc_span::Span; +use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; use rustc_target::spec::abi::{self, Abi}; use std::borrow::Cow; @@ -568,7 +568,7 @@ impl<'tcx> GeneratorSubsts<'tcx> { let layout = tcx.generator_layout(def_id).unwrap(); layout.variant_fields.iter().map(move |variant| { variant.iter().map(move |field| { - ty::EarlyBinder::new(layout.field_tys[*field].ty).subst(tcx, self.substs) + ty::EarlyBinder::bind(layout.field_tys[*field].ty).subst(tcx, self.substs) }) }) } @@ -1459,6 +1459,103 @@ impl ParamConst { #[rustc_pass_by_value] pub struct Region<'tcx>(pub Interned<'tcx, RegionKind<'tcx>>); +impl<'tcx> Region<'tcx> { + #[inline] + pub fn new_early_bound( + tcx: TyCtxt<'tcx>, + early_bound_region: ty::EarlyBoundRegion, + ) -> Region<'tcx> { + tcx.intern_region(ty::ReEarlyBound(early_bound_region)) + } + + #[inline] + pub fn new_late_bound( + tcx: TyCtxt<'tcx>, + debruijn: ty::DebruijnIndex, + bound_region: ty::BoundRegion, + ) -> Region<'tcx> { + // Use a pre-interned one when possible. + if let ty::BoundRegion { var, kind: ty::BrAnon(None) } = bound_region + && let Some(inner) = tcx.lifetimes.re_late_bounds.get(debruijn.as_usize()) + && let Some(re) = inner.get(var.as_usize()).copied() + { + re + } else { + tcx.intern_region(ty::ReLateBound(debruijn, bound_region)) + } + } + + #[inline] + pub fn new_free( + tcx: TyCtxt<'tcx>, + scope: DefId, + bound_region: ty::BoundRegionKind, + ) -> Region<'tcx> { + tcx.intern_region(ty::ReFree(ty::FreeRegion { scope, bound_region })) + } + + #[inline] + pub fn new_var(tcx: TyCtxt<'tcx>, v: ty::RegionVid) -> Region<'tcx> { + // Use a pre-interned one when possible. + tcx.lifetimes + .re_vars + .get(v.as_usize()) + .copied() + .unwrap_or_else(|| tcx.intern_region(ty::ReVar(v))) + } + + #[inline] + pub fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderRegion) -> Region<'tcx> { + tcx.intern_region(ty::RePlaceholder(placeholder)) + } + + /// Constructs a `RegionKind::ReError` region. + #[track_caller] + pub fn new_error(tcx: TyCtxt<'tcx>, reported: ErrorGuaranteed) -> Region<'tcx> { + tcx.intern_region(ty::ReError(reported)) + } + + /// Constructs a `RegionKind::ReError` region and registers a `delay_span_bug` to ensure it + /// gets used. + #[track_caller] + pub fn new_error_misc(tcx: TyCtxt<'tcx>) -> Region<'tcx> { + Region::new_error_with_message( + tcx, + DUMMY_SP, + "RegionKind::ReError constructed but no error reported", + ) + } + + /// Constructs a `RegionKind::ReError` region and registers a `delay_span_bug` with the given + /// `msg` to ensure it gets used. + #[track_caller] + pub fn new_error_with_message<S: Into<MultiSpan>>( + tcx: TyCtxt<'tcx>, + span: S, + msg: &'static str, + ) -> Region<'tcx> { + let reported = tcx.sess.delay_span_bug(span, msg); + Region::new_error(tcx, reported) + } + + /// Avoid this in favour of more specific `new_*` methods, where possible, + /// to avoid the cost of the `match`. + pub fn new_from_kind(tcx: TyCtxt<'tcx>, kind: RegionKind<'tcx>) -> Region<'tcx> { + match kind { + ty::ReEarlyBound(region) => Region::new_early_bound(tcx, region), + ty::ReLateBound(debruijn, region) => Region::new_late_bound(tcx, debruijn, region), + ty::ReFree(ty::FreeRegion { scope, bound_region }) => { + Region::new_free(tcx, scope, bound_region) + } + ty::ReStatic => tcx.lifetimes.re_static, + ty::ReVar(vid) => Region::new_var(tcx, vid), + ty::RePlaceholder(region) => Region::new_placeholder(tcx, region), + ty::ReErased => tcx.lifetimes.re_erased, + ty::ReError(reported) => Region::new_error(tcx, reported), + } + } +} + impl<'tcx> Deref for Region<'tcx> { type Target = RegionKind<'tcx>; @@ -1511,10 +1608,11 @@ impl Atom for RegionVid { rustc_index::newtype_index! { #[derive(HashStable)] + #[debug_format = "{}"] pub struct BoundVar {} } -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)] #[derive(HashStable)] pub struct BoundTy { pub var: BoundVar, diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs index 8691a3070bc..8685a22d9ca 100644 --- a/compiler/rustc_middle/src/ty/subst.rs +++ b/compiler/rustc_middle/src/ty/subst.rs @@ -545,7 +545,7 @@ impl<'tcx, T> !TypeFoldable<TyCtxt<'tcx>> for ty::EarlyBinder<T> {} impl<'tcx, T> !TypeVisitable<TyCtxt<'tcx>> for ty::EarlyBinder<T> {} impl<T> EarlyBinder<T> { - pub fn new(inner: T) -> EarlyBinder<T> { + pub fn bind(inner: T) -> EarlyBinder<T> { EarlyBinder(inner) } diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index dade50df5d9..311f63adc7f 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -709,7 +709,7 @@ impl<'tcx> TyCtxt<'tcx> { .as_ref() .map_or_else(|| [].iter(), |l| l.field_tys.iter()) .filter(|decl| !decl.ignore_for_traits) - .map(|decl| ty::EarlyBinder::new(decl.ty)) + .map(|decl| ty::EarlyBinder::bind(decl.ty)) } /// Normalizes all opaque types in the given value, replacing them diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs index 58374f6b3bd..a00c77ccbcd 100644 --- a/compiler/rustc_middle/src/values.rs +++ b/compiler/rustc_middle/src/values.rs @@ -96,13 +96,13 @@ impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for Representability { impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for ty::EarlyBinder<Ty<'_>> { fn from_cycle_error(tcx: TyCtxt<'tcx>, cycle: &[QueryInfo<DepKind>]) -> Self { - ty::EarlyBinder::new(Ty::from_cycle_error(tcx, cycle)) + ty::EarlyBinder::bind(Ty::from_cycle_error(tcx, cycle)) } } impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for ty::EarlyBinder<ty::Binder<'_, ty::FnSig<'_>>> { fn from_cycle_error(tcx: TyCtxt<'tcx>, cycle: &[QueryInfo<DepKind>]) -> Self { - ty::EarlyBinder::new(ty::Binder::from_cycle_error(tcx, cycle)) + ty::EarlyBinder::bind(ty::Binder::from_cycle_error(tcx, cycle)) } } diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs index 463f639defe..d00fb754c64 100644 --- a/compiler/rustc_mir_build/src/thir/cx/mod.rs +++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs @@ -142,7 +142,7 @@ impl<'tcx> Cx<'tcx> { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BrEnv, }; - let env_region = self.tcx.mk_re_late_bound(ty::INNERMOST, br); + let env_region = ty::Region::new_late_bound(self.tcx, ty::INNERMOST, br); let closure_env_ty = self.tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap(); let liberated_closure_env_ty = self.tcx.erase_late_bound_regions( diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index aeca0073304..6ae6bdc17d5 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -199,8 +199,7 @@ impl DefUse { | NonMutatingUseContext::Move | NonMutatingUseContext::PlaceMention | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::SharedBorrow - | NonMutatingUseContext::UniqueBorrow, + | NonMutatingUseContext::SharedBorrow, ) => Some(DefUse::Use), PlaceContext::MutatingUse(MutatingUseContext::Projection) diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index a5d18fff89b..1ba1951afde 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -772,7 +772,6 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // mutation. | NonMutatingUse(NonMutatingUseContext::SharedBorrow) | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) - | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | NonMutatingUse(NonMutatingUseContext::AddressOf) | MutatingUse(MutatingUseContext::Borrow) | MutatingUse(MutatingUseContext::AddressOf) => { diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 319f3a79705..3df459dfa79 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -130,7 +130,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { PlaceContext::NonMutatingUse( NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::UniqueBorrow | NonMutatingUseContext::AddressOf, ) => true, // For debuginfo, merging locals is ok. diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index c0458126840..b1c9c4acc40 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -83,7 +83,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { // If the inner type matches the type bound by `Pointer` if inner_ty == bound_ty { // Do a substitution using the parameters from the callsite - let subst_ty = EarlyBinder::new(inner_ty).subst(self.tcx, substs_ref); + let subst_ty = EarlyBinder::bind(inner_ty).subst(self.tcx, substs_ref); if let Some((fn_id, fn_substs)) = FunctionItemRefChecker::is_fn_ref(subst_ty) { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index e27d48fa8f6..55b9f084c39 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -192,7 +192,7 @@ impl<'tcx> Inliner<'tcx> { let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, - ty::EarlyBinder::new(callee_body.clone()), + ty::EarlyBinder::bind(callee_body.clone()), ) else { return Err("failed to normalize callee body"); }; @@ -455,7 +455,7 @@ impl<'tcx> Inliner<'tcx> { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = callsite .callee - .subst_mir(self.tcx, ty::EarlyBinder::new(&place.ty(callee_body, tcx).ty)); + .subst_mir(self.tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } @@ -790,7 +790,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = self .instance - .subst_mir(tcx, ty::EarlyBinder::new(&place.ty(self.callee_body, tcx).ty)); + .subst_mir(tcx, ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -801,7 +801,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder::new(&f.literal.ty())); + let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder::bind(&f.literal.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 00842e9d665..8a10445f837 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -47,7 +47,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( let Ok(substs) = caller.try_subst_mir_and_normalize_erasing_regions( tcx, param_env, - ty::EarlyBinder::new(substs), + ty::EarlyBinder::bind(substs), ) else { trace!(?caller, ?param_env, ?substs, "cannot normalize, skipping"); continue; diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 522220f3d77..ae726dea944 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -69,7 +69,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' // of this function. Is this intentional? if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(Ty::kind) { let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); - let body = EarlyBinder::new(body.clone()).subst(tcx, substs); + let body = EarlyBinder::bind(body.clone()).subst(tcx, substs); debug!("make_shim({:?}) = {:?}", instance, body); return body; } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index e8e4246b797..7a0d3a025f3 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -216,7 +216,6 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor<'_> { PlaceContext::NonMutatingUse( NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::UniqueBorrow | NonMutatingUseContext::AddressOf, ) | PlaceContext::MutatingUse(_) => { diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 8874aa7d3ca..cefa64d27ac 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -677,7 +677,7 @@ impl<'a, 'tcx> MirNeighborCollector<'a, 'tcx> { self.instance.subst_mir_and_normalize_erasing_regions( self.tcx, ty::ParamEnv::reveal_all(), - ty::EarlyBinder::new(value), + ty::EarlyBinder::bind(value), ) } } diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs index ecc50c3f664..89dadc782f2 100644 --- a/compiler/rustc_monomorphize/src/lib.rs +++ b/compiler/rustc_monomorphize/src/lib.rs @@ -1,4 +1,5 @@ #![feature(array_windows)] +#![feature(is_sorted)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs new file mode 100644 index 00000000000..1b872484eb1 --- /dev/null +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -0,0 +1,1194 @@ +//! Partitioning Codegen Units for Incremental Compilation +//! ====================================================== +//! +//! The task of this module is to take the complete set of monomorphizations of +//! a crate and produce a set of codegen units from it, where a codegen unit +//! is a named set of (mono-item, linkage) pairs. That is, this module +//! decides which monomorphization appears in which codegen units with which +//! linkage. The following paragraphs describe some of the background on the +//! partitioning scheme. +//! +//! The most important opportunity for saving on compilation time with +//! incremental compilation is to avoid re-codegenning and re-optimizing code. +//! Since the unit of codegen and optimization for LLVM is "modules" or, how +//! we call them "codegen units", the particulars of how much time can be saved +//! by incremental compilation are tightly linked to how the output program is +//! partitioned into these codegen units prior to passing it to LLVM -- +//! especially because we have to treat codegen units as opaque entities once +//! they are created: There is no way for us to incrementally update an existing +//! LLVM module and so we have to build any such module from scratch if it was +//! affected by some change in the source code. +//! +//! From that point of view it would make sense to maximize the number of +//! codegen units by, for example, putting each function into its own module. +//! That way only those modules would have to be re-compiled that were actually +//! affected by some change, minimizing the number of functions that could have +//! been re-used but just happened to be located in a module that is +//! re-compiled. +//! +//! However, since LLVM optimization does not work across module boundaries, +//! using such a highly granular partitioning would lead to very slow runtime +//! code since it would effectively prohibit inlining and other inter-procedure +//! optimizations. We want to avoid that as much as possible. +//! +//! Thus we end up with a trade-off: The bigger the codegen units, the better +//! LLVM's optimizer can do its work, but also the smaller the compilation time +//! reduction we get from incremental compilation. +//! +//! Ideally, we would create a partitioning such that there are few big codegen +//! units with few interdependencies between them. For now though, we use the +//! following heuristic to determine the partitioning: +//! +//! - There are two codegen units for every source-level module: +//! - One for "stable", that is non-generic, code +//! - One for more "volatile" code, i.e., monomorphized instances of functions +//! defined in that module +//! +//! In order to see why this heuristic makes sense, let's take a look at when a +//! codegen unit can get invalidated: +//! +//! 1. The most straightforward case is when the BODY of a function or global +//! changes. Then any codegen unit containing the code for that item has to be +//! re-compiled. Note that this includes all codegen units where the function +//! has been inlined. +//! +//! 2. The next case is when the SIGNATURE of a function or global changes. In +//! this case, all codegen units containing a REFERENCE to that item have to be +//! re-compiled. This is a superset of case 1. +//! +//! 3. The final and most subtle case is when a REFERENCE to a generic function +//! is added or removed somewhere. Even though the definition of the function +//! might be unchanged, a new REFERENCE might introduce a new monomorphized +//! instance of this function which has to be placed and compiled somewhere. +//! Conversely, when removing a REFERENCE, it might have been the last one with +//! that particular set of generic arguments and thus we have to remove it. +//! +//! From the above we see that just using one codegen unit per source-level +//! module is not such a good idea, since just adding a REFERENCE to some +//! generic item somewhere else would invalidate everything within the module +//! containing the generic item. The heuristic above reduces this detrimental +//! side-effect of references a little by at least not touching the non-generic +//! code of the module. +//! +//! A Note on Inlining +//! ------------------ +//! As briefly mentioned above, in order for LLVM to be able to inline a +//! function call, the body of the function has to be available in the LLVM +//! module where the call is made. This has a few consequences for partitioning: +//! +//! - The partitioning algorithm has to take care of placing functions into all +//! codegen units where they should be available for inlining. It also has to +//! decide on the correct linkage for these functions. +//! +//! - The partitioning algorithm has to know which functions are likely to get +//! inlined, so it can distribute function instantiations accordingly. Since +//! there is no way of knowing for sure which functions LLVM will decide to +//! inline in the end, we apply a heuristic here: Only functions marked with +//! `#[inline]` are considered for inlining by the partitioner. The current +//! implementation will not try to determine if a function is likely to be +//! inlined by looking at the functions definition. +//! +//! Note though that as a side-effect of creating a codegen units per +//! source-level module, functions from the same module will be available for +//! inlining, even when they are not marked `#[inline]`. + +use std::cmp; +use std::collections::hash_map::Entry; +use std::fs::{self, File}; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::sync; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE}; +use rustc_hir::definitions::DefPathDataName; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; +use rustc_middle::mir; +use rustc_middle::mir::mono::{ + CodegenUnit, CodegenUnitNameBuilder, InstantiationMode, Linkage, MonoItem, Visibility, +}; +use rustc_middle::query::Providers; +use rustc_middle::ty::print::{characteristic_def_id_of_type, with_no_trimmed_paths}; +use rustc_middle::ty::{self, visit::TypeVisitableExt, InstanceDef, TyCtxt}; +use rustc_session::config::{DumpMonoStatsFormat, SwitchWithOptPath}; +use rustc_span::symbol::Symbol; + +use crate::collector::InliningMap; +use crate::collector::{self, MonoItemCollectionMode}; +use crate::errors::{CouldntDumpMonoStats, SymbolAlreadyDefined, UnknownCguCollectionMode}; + +struct PartitioningCx<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + target_cgu_count: usize, + inlining_map: &'a InliningMap<'tcx>, +} + +struct PlacedRootMonoItems<'tcx> { + /// The codegen units, sorted by name to make things deterministic. + codegen_units: Vec<CodegenUnit<'tcx>>, + + roots: FxHashSet<MonoItem<'tcx>>, + internalization_candidates: FxHashSet<MonoItem<'tcx>>, +} + +// The output CGUs are sorted by name. +fn partition<'tcx, I>( + tcx: TyCtxt<'tcx>, + mono_items: &mut I, + max_cgu_count: usize, + inlining_map: &InliningMap<'tcx>, +) -> Vec<CodegenUnit<'tcx>> +where + I: Iterator<Item = MonoItem<'tcx>>, +{ + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning"); + + let cx = &PartitioningCx { tcx, target_cgu_count: max_cgu_count, inlining_map }; + + // In the first step, we place all regular monomorphizations into their + // respective 'home' codegen unit. Regular monomorphizations are all + // functions and statics defined in the local crate. + let PlacedRootMonoItems { mut codegen_units, roots, internalization_candidates } = { + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_roots"); + place_root_mono_items(cx, mono_items) + }; + + for cgu in &mut codegen_units { + cgu.create_size_estimate(tcx); + } + + debug_dump(tcx, "INITIAL PARTITIONING", &codegen_units); + + // Merge until we have at most `max_cgu_count` codegen units. + // `merge_codegen_units` is responsible for updating the CGU size + // estimates. + { + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_merge_cgus"); + merge_codegen_units(cx, &mut codegen_units); + debug_dump(tcx, "POST MERGING", &codegen_units); + } + + // In the next step, we use the inlining map to determine which additional + // monomorphizations have to go into each codegen unit. These additional + // monomorphizations can be drop-glue, functions from external crates, and + // local functions the definition of which is marked with `#[inline]`. + let mono_item_placements = { + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_inline_items"); + place_inlined_mono_items(cx, &mut codegen_units, roots) + }; + + for cgu in &mut codegen_units { + cgu.create_size_estimate(tcx); + } + + debug_dump(tcx, "POST INLINING", &codegen_units); + + // Next we try to make as many symbols "internal" as possible, so LLVM has + // more freedom to optimize. + if !tcx.sess.link_dead_code() { + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols"); + internalize_symbols( + cx, + &mut codegen_units, + mono_item_placements, + internalization_candidates, + ); + } + + let instrument_dead_code = + tcx.sess.instrument_coverage() && !tcx.sess.instrument_coverage_except_unused_functions(); + + if instrument_dead_code { + assert!( + codegen_units.len() > 0, + "There must be at least one CGU that code coverage data can be generated in." + ); + + // Find the smallest CGU that has exported symbols and put the dead + // function stubs in that CGU. We look for exported symbols to increase + // the likelihood the linker won't throw away the dead functions. + // FIXME(#92165): In order to truly resolve this, we need to make sure + // the object file (CGU) containing the dead function stubs is included + // in the final binary. This will probably require forcing these + // function symbols to be included via `-u` or `/include` linker args. + let mut cgus: Vec<_> = codegen_units.iter_mut().collect(); + cgus.sort_by_key(|cgu| cgu.size_estimate()); + + let dead_code_cgu = + if let Some(cgu) = cgus.into_iter().rev().find(|cgu| { + cgu.items().iter().any(|(_, (linkage, _))| *linkage == Linkage::External) + }) { + cgu + } else { + // If there are no CGUs that have externally linked items, + // then we just pick the first CGU as a fallback. + &mut codegen_units[0] + }; + dead_code_cgu.make_code_coverage_dead_code_cgu(); + } + + // Ensure CGUs are sorted by name, so that we get deterministic results. + assert!(codegen_units.is_sorted_by(|a, b| Some(a.name().as_str().cmp(b.name().as_str())))); + + debug_dump(tcx, "FINAL", &codegen_units); + + codegen_units +} + +fn place_root_mono_items<'tcx, I>( + cx: &PartitioningCx<'_, 'tcx>, + mono_items: &mut I, +) -> PlacedRootMonoItems<'tcx> +where + I: Iterator<Item = MonoItem<'tcx>>, +{ + let mut roots = FxHashSet::default(); + let mut codegen_units = FxHashMap::default(); + let is_incremental_build = cx.tcx.sess.opts.incremental.is_some(); + let mut internalization_candidates = FxHashSet::default(); + + // Determine if monomorphizations instantiated in this crate will be made + // available to downstream crates. This depends on whether we are in + // share-generics mode and whether the current crate can even have + // downstream crates. + let export_generics = + cx.tcx.sess.opts.share_generics() && cx.tcx.local_crate_exports_generics(); + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx); + let cgu_name_cache = &mut FxHashMap::default(); + + for mono_item in mono_items { + match mono_item.instantiation_mode(cx.tcx) { + InstantiationMode::GloballyShared { .. } => {} + InstantiationMode::LocalCopy => continue, + } + + let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item); + let is_volatile = is_incremental_build && mono_item.is_generic_fn(); + + let codegen_unit_name = match characteristic_def_id { + Some(def_id) => compute_codegen_unit_name( + cx.tcx, + cgu_name_builder, + def_id, + is_volatile, + cgu_name_cache, + ), + None => fallback_cgu_name(cgu_name_builder), + }; + + let codegen_unit = codegen_units + .entry(codegen_unit_name) + .or_insert_with(|| CodegenUnit::new(codegen_unit_name)); + + let mut can_be_internalized = true; + let (linkage, visibility) = mono_item_linkage_and_visibility( + cx.tcx, + &mono_item, + &mut can_be_internalized, + export_generics, + ); + if visibility == Visibility::Hidden && can_be_internalized { + internalization_candidates.insert(mono_item); + } + + codegen_unit.items_mut().insert(mono_item, (linkage, visibility)); + roots.insert(mono_item); + } + + // Always ensure we have at least one CGU; otherwise, if we have a + // crate with just types (for example), we could wind up with no CGU. + if codegen_units.is_empty() { + let codegen_unit_name = fallback_cgu_name(cgu_name_builder); + codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name)); + } + + let mut codegen_units: Vec<_> = codegen_units.into_values().collect(); + codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); + + PlacedRootMonoItems { codegen_units, roots, internalization_candidates } +} + +// This function requires the CGUs to be sorted by name on input, and ensures +// they are sorted by name on return, for deterministic behaviour. +fn merge_codegen_units<'tcx>( + cx: &PartitioningCx<'_, 'tcx>, + codegen_units: &mut Vec<CodegenUnit<'tcx>>, +) { + assert!(cx.target_cgu_count >= 1); + + // A sorted order here ensures merging is deterministic. + assert!(codegen_units.is_sorted_by(|a, b| Some(a.name().as_str().cmp(b.name().as_str())))); + + // This map keeps track of what got merged into what. + let mut cgu_contents: FxHashMap<Symbol, Vec<Symbol>> = + codegen_units.iter().map(|cgu| (cgu.name(), vec![cgu.name()])).collect(); + + // Merge the two smallest codegen units until the target size is + // reached. + while codegen_units.len() > cx.target_cgu_count { + // Sort small cgus to the back + codegen_units.sort_by_cached_key(|cgu| cmp::Reverse(cgu.size_estimate())); + let mut smallest = codegen_units.pop().unwrap(); + let second_smallest = codegen_units.last_mut().unwrap(); + + // Move the mono-items from `smallest` to `second_smallest` + second_smallest.modify_size_estimate(smallest.size_estimate()); + for (k, v) in smallest.items_mut().drain() { + second_smallest.items_mut().insert(k, v); + } + + // Record that `second_smallest` now contains all the stuff that was + // in `smallest` before. + let mut consumed_cgu_names = cgu_contents.remove(&smallest.name()).unwrap(); + cgu_contents.get_mut(&second_smallest.name()).unwrap().append(&mut consumed_cgu_names); + + debug!( + "CodegenUnit {} merged into CodegenUnit {}", + smallest.name(), + second_smallest.name() + ); + } + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx); + + if cx.tcx.sess.opts.incremental.is_some() { + // If we are doing incremental compilation, we want CGU names to + // reflect the path of the source level module they correspond to. + // For CGUs that contain the code of multiple modules because of the + // merging done above, we use a concatenation of the names of all + // contained CGUs. + let new_cgu_names: FxHashMap<Symbol, String> = cgu_contents + .into_iter() + // This `filter` makes sure we only update the name of CGUs that + // were actually modified by merging. + .filter(|(_, cgu_contents)| cgu_contents.len() > 1) + .map(|(current_cgu_name, cgu_contents)| { + let mut cgu_contents: Vec<&str> = cgu_contents.iter().map(|s| s.as_str()).collect(); + + // Sort the names, so things are deterministic and easy to + // predict. We are sorting primitive `&str`s here so we can + // use unstable sort. + cgu_contents.sort_unstable(); + + (current_cgu_name, cgu_contents.join("--")) + }) + .collect(); + + for cgu in codegen_units.iter_mut() { + if let Some(new_cgu_name) = new_cgu_names.get(&cgu.name()) { + if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names { + cgu.set_name(Symbol::intern(&new_cgu_name)); + } else { + // If we don't require CGU names to be human-readable, + // we use a fixed length hash of the composite CGU name + // instead. + let new_cgu_name = CodegenUnit::mangle_name(&new_cgu_name); + cgu.set_name(Symbol::intern(&new_cgu_name)); + } + } + } + } else { + // If we are compiling non-incrementally we just generate simple CGU + // names containing an index. + for (index, cgu) in codegen_units.iter_mut().enumerate() { + let numbered_codegen_unit_name = + cgu_name_builder.build_cgu_name_no_mangle(LOCAL_CRATE, &["cgu"], Some(index)); + cgu.set_name(numbered_codegen_unit_name); + } + } + + // A sorted order here ensures what follows can be deterministic. + codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); +} + +/// For symbol internalization, we need to know whether a symbol/mono-item is +/// accessed from outside the codegen unit it is defined in. This type is used +/// to keep track of that. +#[derive(Clone, PartialEq, Eq, Debug)] +enum MonoItemPlacement { + SingleCgu { cgu_name: Symbol }, + MultipleCgus, +} + +fn place_inlined_mono_items<'tcx>( + cx: &PartitioningCx<'_, 'tcx>, + codegen_units: &mut [CodegenUnit<'tcx>], + roots: FxHashSet<MonoItem<'tcx>>, +) -> FxHashMap<MonoItem<'tcx>, MonoItemPlacement> { + let mut mono_item_placements = FxHashMap::default(); + + let single_codegen_unit = codegen_units.len() == 1; + + for old_codegen_unit in codegen_units.iter_mut() { + // Collect all items that need to be available in this codegen unit. + let mut reachable = FxHashSet::default(); + for root in old_codegen_unit.items().keys() { + follow_inlining(*root, cx.inlining_map, &mut reachable); + } + + let mut new_codegen_unit = CodegenUnit::new(old_codegen_unit.name()); + + // Add all monomorphizations that are not already there. + for mono_item in reachable { + if let Some(linkage) = old_codegen_unit.items().get(&mono_item) { + // This is a root, just copy it over. + new_codegen_unit.items_mut().insert(mono_item, *linkage); + } else { + if roots.contains(&mono_item) { + bug!( + "GloballyShared mono-item inlined into other CGU: \ + {:?}", + mono_item + ); + } + + // This is a CGU-private copy. + new_codegen_unit + .items_mut() + .insert(mono_item, (Linkage::Internal, Visibility::Default)); + } + + if !single_codegen_unit { + // If there is more than one codegen unit, we need to keep track + // in which codegen units each monomorphization is placed. + match mono_item_placements.entry(mono_item) { + Entry::Occupied(e) => { + let placement = e.into_mut(); + debug_assert!(match *placement { + MonoItemPlacement::SingleCgu { cgu_name } => { + cgu_name != new_codegen_unit.name() + } + MonoItemPlacement::MultipleCgus => true, + }); + *placement = MonoItemPlacement::MultipleCgus; + } + Entry::Vacant(e) => { + e.insert(MonoItemPlacement::SingleCgu { + cgu_name: new_codegen_unit.name(), + }); + } + } + } + } + + *old_codegen_unit = new_codegen_unit; + } + + return mono_item_placements; + + fn follow_inlining<'tcx>( + mono_item: MonoItem<'tcx>, + inlining_map: &InliningMap<'tcx>, + visited: &mut FxHashSet<MonoItem<'tcx>>, + ) { + if !visited.insert(mono_item) { + return; + } + + inlining_map.with_inlining_candidates(mono_item, |target| { + follow_inlining(target, inlining_map, visited); + }); + } +} + +fn internalize_symbols<'tcx>( + cx: &PartitioningCx<'_, 'tcx>, + codegen_units: &mut [CodegenUnit<'tcx>], + mono_item_placements: FxHashMap<MonoItem<'tcx>, MonoItemPlacement>, + internalization_candidates: FxHashSet<MonoItem<'tcx>>, +) { + if codegen_units.len() == 1 { + // Fast path for when there is only one codegen unit. In this case we + // can internalize all candidates, since there is nowhere else they + // could be accessed from. + for cgu in codegen_units { + for candidate in &internalization_candidates { + cgu.items_mut().insert(*candidate, (Linkage::Internal, Visibility::Default)); + } + } + + return; + } + + // Build a map from every monomorphization to all the monomorphizations that + // reference it. + let mut accessor_map: FxHashMap<MonoItem<'tcx>, Vec<MonoItem<'tcx>>> = Default::default(); + cx.inlining_map.iter_accesses(|accessor, accessees| { + for accessee in accessees { + accessor_map.entry(*accessee).or_default().push(accessor); + } + }); + + // For each internalization candidates in each codegen unit, check if it is + // accessed from outside its defining codegen unit. + for cgu in codegen_units { + let home_cgu = MonoItemPlacement::SingleCgu { cgu_name: cgu.name() }; + + for (accessee, linkage_and_visibility) in cgu.items_mut() { + if !internalization_candidates.contains(accessee) { + // This item is no candidate for internalizing, so skip it. + continue; + } + debug_assert_eq!(mono_item_placements[accessee], home_cgu); + + if let Some(accessors) = accessor_map.get(accessee) { + if accessors + .iter() + .filter_map(|accessor| { + // Some accessors might not have been + // instantiated. We can safely ignore those. + mono_item_placements.get(accessor) + }) + .any(|placement| *placement != home_cgu) + { + // Found an accessor from another CGU, so skip to the next + // item without marking this one as internal. + continue; + } + } + + // If we got here, we did not find any accesses from other CGUs, + // so it's fine to make this monomorphization internal. + *linkage_and_visibility = (Linkage::Internal, Visibility::Default); + } + } +} + +fn characteristic_def_id_of_mono_item<'tcx>( + tcx: TyCtxt<'tcx>, + mono_item: MonoItem<'tcx>, +) -> Option<DefId> { + match mono_item { + MonoItem::Fn(instance) => { + let def_id = match instance.def { + ty::InstanceDef::Item(def) => def, + ty::InstanceDef::VTableShim(..) + | ty::InstanceDef::ReifyShim(..) + | ty::InstanceDef::FnPtrShim(..) + | ty::InstanceDef::ClosureOnceShim { .. } + | ty::InstanceDef::Intrinsic(..) + | ty::InstanceDef::DropGlue(..) + | ty::InstanceDef::Virtual(..) + | ty::InstanceDef::CloneShim(..) + | ty::InstanceDef::ThreadLocalShim(..) + | ty::InstanceDef::FnPtrAddrShim(..) => return None, + }; + + // If this is a method, we want to put it into the same module as + // its self-type. If the self-type does not provide a characteristic + // DefId, we use the location of the impl after all. + + if tcx.trait_of_item(def_id).is_some() { + let self_ty = instance.substs.type_at(0); + // This is a default implementation of a trait method. + return characteristic_def_id_of_type(self_ty).or(Some(def_id)); + } + + if let Some(impl_def_id) = tcx.impl_of_method(def_id) { + if tcx.sess.opts.incremental.is_some() + && tcx.trait_id_of_impl(impl_def_id) == tcx.lang_items().drop_trait() + { + // Put `Drop::drop` into the same cgu as `drop_in_place` + // since `drop_in_place` is the only thing that can + // call it. + return None; + } + + // When polymorphization is enabled, methods which do not depend on their generic + // parameters, but the self-type of their impl block do will fail to normalize. + if !tcx.sess.opts.unstable_opts.polymorphize || !instance.has_param() { + // This is a method within an impl, find out what the self-type is: + let impl_self_ty = tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + tcx.type_of(impl_def_id), + ); + if let Some(def_id) = characteristic_def_id_of_type(impl_self_ty) { + return Some(def_id); + } + } + } + + Some(def_id) + } + MonoItem::Static(def_id) => Some(def_id), + MonoItem::GlobalAsm(item_id) => Some(item_id.owner_id.to_def_id()), + } +} + +fn compute_codegen_unit_name( + tcx: TyCtxt<'_>, + name_builder: &mut CodegenUnitNameBuilder<'_>, + def_id: DefId, + volatile: bool, + cache: &mut CguNameCache, +) -> Symbol { + // Find the innermost module that is not nested within a function. + let mut current_def_id = def_id; + let mut cgu_def_id = None; + // Walk backwards from the item we want to find the module for. + loop { + if current_def_id.is_crate_root() { + if cgu_def_id.is_none() { + // If we have not found a module yet, take the crate root. + cgu_def_id = Some(def_id.krate.as_def_id()); + } + break; + } else if tcx.def_kind(current_def_id) == DefKind::Mod { + if cgu_def_id.is_none() { + cgu_def_id = Some(current_def_id); + } + } else { + // If we encounter something that is not a module, throw away + // any module that we've found so far because we now know that + // it is nested within something else. + cgu_def_id = None; + } + + current_def_id = tcx.parent(current_def_id); + } + + let cgu_def_id = cgu_def_id.unwrap(); + + *cache.entry((cgu_def_id, volatile)).or_insert_with(|| { + let def_path = tcx.def_path(cgu_def_id); + + let components = def_path.data.iter().map(|part| match part.data.name() { + DefPathDataName::Named(name) => name, + DefPathDataName::Anon { .. } => unreachable!(), + }); + + let volatile_suffix = volatile.then_some("volatile"); + + name_builder.build_cgu_name(def_path.krate, components, volatile_suffix) + }) +} + +// Anything we can't find a proper codegen unit for goes into this. +fn fallback_cgu_name(name_builder: &mut CodegenUnitNameBuilder<'_>) -> Symbol { + name_builder.build_cgu_name(LOCAL_CRATE, &["fallback"], Some("cgu")) +} + +fn mono_item_linkage_and_visibility<'tcx>( + tcx: TyCtxt<'tcx>, + mono_item: &MonoItem<'tcx>, + can_be_internalized: &mut bool, + export_generics: bool, +) -> (Linkage, Visibility) { + if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) { + return (explicit_linkage, Visibility::Default); + } + let vis = mono_item_visibility(tcx, mono_item, can_be_internalized, export_generics); + (Linkage::External, vis) +} + +type CguNameCache = FxHashMap<(DefId, bool), Symbol>; + +fn static_visibility<'tcx>( + tcx: TyCtxt<'tcx>, + can_be_internalized: &mut bool, + def_id: DefId, +) -> Visibility { + if tcx.is_reachable_non_generic(def_id) { + *can_be_internalized = false; + default_visibility(tcx, def_id, false) + } else { + Visibility::Hidden + } +} + +fn mono_item_visibility<'tcx>( + tcx: TyCtxt<'tcx>, + mono_item: &MonoItem<'tcx>, + can_be_internalized: &mut bool, + export_generics: bool, +) -> Visibility { + let instance = match mono_item { + // This is pretty complicated; see below. + MonoItem::Fn(instance) => instance, + + // Misc handling for generics and such, but otherwise: + MonoItem::Static(def_id) => return static_visibility(tcx, can_be_internalized, *def_id), + MonoItem::GlobalAsm(item_id) => { + return static_visibility(tcx, can_be_internalized, item_id.owner_id.to_def_id()); + } + }; + + let def_id = match instance.def { + InstanceDef::Item(def_id) | InstanceDef::DropGlue(def_id, Some(_)) => def_id, + + // We match the visibility of statics here + InstanceDef::ThreadLocalShim(def_id) => { + return static_visibility(tcx, can_be_internalized, def_id); + } + + // These are all compiler glue and such, never exported, always hidden. + InstanceDef::VTableShim(..) + | InstanceDef::ReifyShim(..) + | InstanceDef::FnPtrShim(..) + | InstanceDef::Virtual(..) + | InstanceDef::Intrinsic(..) + | InstanceDef::ClosureOnceShim { .. } + | InstanceDef::DropGlue(..) + | InstanceDef::CloneShim(..) + | InstanceDef::FnPtrAddrShim(..) => return Visibility::Hidden, + }; + + // The `start_fn` lang item is actually a monomorphized instance of a + // function in the standard library, used for the `main` function. We don't + // want to export it so we tag it with `Hidden` visibility but this symbol + // is only referenced from the actual `main` symbol which we unfortunately + // don't know anything about during partitioning/collection. As a result we + // forcibly keep this symbol out of the `internalization_candidates` set. + // + // FIXME: eventually we don't want to always force this symbol to have + // hidden visibility, it should indeed be a candidate for + // internalization, but we have to understand that it's referenced + // from the `main` symbol we'll generate later. + // + // This may be fixable with a new `InstanceDef` perhaps? Unsure! + if tcx.lang_items().start_fn() == Some(def_id) { + *can_be_internalized = false; + return Visibility::Hidden; + } + + let is_generic = instance.substs.non_erasable_generics().next().is_some(); + + // Upstream `DefId` instances get different handling than local ones. + let Some(def_id) = def_id.as_local() else { + return if export_generics && is_generic { + // If it is an upstream monomorphization and we export generics, we must make + // it available to downstream crates. + *can_be_internalized = false; + default_visibility(tcx, def_id, true) + } else { + Visibility::Hidden + }; + }; + + if is_generic { + if export_generics { + if tcx.is_unreachable_local_definition(def_id) { + // This instance cannot be used from another crate. + Visibility::Hidden + } else { + // This instance might be useful in a downstream crate. + *can_be_internalized = false; + default_visibility(tcx, def_id.to_def_id(), true) + } + } else { + // We are not exporting generics or the definition is not reachable + // for downstream crates, we can internalize its instantiations. + Visibility::Hidden + } + } else { + // If this isn't a generic function then we mark this a `Default` if + // this is a reachable item, meaning that it's a symbol other crates may + // access when they link to us. + if tcx.is_reachable_non_generic(def_id.to_def_id()) { + *can_be_internalized = false; + debug_assert!(!is_generic); + return default_visibility(tcx, def_id.to_def_id(), false); + } + + // If this isn't reachable then we're gonna tag this with `Hidden` + // visibility. In some situations though we'll want to prevent this + // symbol from being internalized. + // + // There's two categories of items here: + // + // * First is weak lang items. These are basically mechanisms for + // libcore to forward-reference symbols defined later in crates like + // the standard library or `#[panic_handler]` definitions. The + // definition of these weak lang items needs to be referencable by + // libcore, so we're no longer a candidate for internalization. + // Removal of these functions can't be done by LLVM but rather must be + // done by the linker as it's a non-local decision. + // + // * Second is "std internal symbols". Currently this is primarily used + // for allocator symbols. Allocators are a little weird in their + // implementation, but the idea is that the compiler, at the last + // minute, defines an allocator with an injected object file. The + // `alloc` crate references these symbols (`__rust_alloc`) and the + // definition doesn't get hooked up until a linked crate artifact is + // generated. + // + // The symbols synthesized by the compiler (`__rust_alloc`) are thin + // veneers around the actual implementation, some other symbol which + // implements the same ABI. These symbols (things like `__rg_alloc`, + // `__rdl_alloc`, `__rde_alloc`, etc), are all tagged with "std + // internal symbols". + // + // The std-internal symbols here **should not show up in a dll as an + // exported interface**, so they return `false` from + // `is_reachable_non_generic` above and we'll give them `Hidden` + // visibility below. Like the weak lang items, though, we can't let + // LLVM internalize them as this decision is left up to the linker to + // omit them, so prevent them from being internalized. + let attrs = tcx.codegen_fn_attrs(def_id); + if attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) { + *can_be_internalized = false; + } + + Visibility::Hidden + } +} + +fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibility { + if !tcx.sess.target.default_hidden_visibility { + return Visibility::Default; + } + + // Generic functions never have export-level C. + if is_generic { + return Visibility::Hidden; + } + + // Things with export level C don't get instantiated in + // downstream crates. + if !id.is_local() { + return Visibility::Hidden; + } + + // C-export level items remain at `Default`, all other internal + // items become `Hidden`. + match tcx.reachable_non_generics(id.krate).get(&id) { + Some(SymbolExportInfo { level: SymbolExportLevel::C, .. }) => Visibility::Default, + _ => Visibility::Hidden, + } +} + +fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit<'tcx>]) { + let dump = move || { + use std::fmt::Write; + + let num_cgus = cgus.len(); + let num_items: usize = cgus.iter().map(|cgu| cgu.items().len()).sum(); + let total_size: usize = cgus.iter().map(|cgu| cgu.size_estimate()).sum(); + let max_size = cgus.iter().map(|cgu| cgu.size_estimate()).max().unwrap(); + let min_size = cgus.iter().map(|cgu| cgu.size_estimate()).min().unwrap(); + let max_min_size_ratio = max_size as f64 / min_size as f64; + + let s = &mut String::new(); + let _ = writeln!( + s, + "{label} ({num_items} items, total_size={total_size}; {num_cgus} CGUs, \ + max_size={max_size}, min_size={min_size}, max_size/min_size={max_min_size_ratio:.1}):" + ); + for (i, cgu) in cgus.iter().enumerate() { + let num_items = cgu.items().len(); + let _ = writeln!( + s, + "- CGU[{i}] {} ({num_items} items, size={}):", + cgu.name(), + cgu.size_estimate() + ); + + // The order of `cgu.items()` is non-deterministic; sort it by name + // to give deterministic output. + let mut items: Vec<_> = cgu.items().iter().collect(); + items.sort_by_key(|(item, _)| item.symbol_name(tcx).name); + for (item, linkage) in items { + let symbol_name = item.symbol_name(tcx).name; + let symbol_hash_start = symbol_name.rfind('h'); + let symbol_hash = symbol_hash_start.map_or("<no hash>", |i| &symbol_name[i..]); + + let size = item.size_estimate(tcx); + let _ = with_no_trimmed_paths!(writeln!( + s, + " - {item} [{linkage:?}] [{symbol_hash}] (size={size})" + )); + } + + let _ = writeln!(s); + } + + std::mem::take(s) + }; + + debug!("{}", dump()); +} + +#[inline(never)] // give this a place in the profiler +fn assert_symbols_are_distinct<'a, 'tcx, I>(tcx: TyCtxt<'tcx>, mono_items: I) +where + I: Iterator<Item = &'a MonoItem<'tcx>>, + 'tcx: 'a, +{ + let _prof_timer = tcx.prof.generic_activity("assert_symbols_are_distinct"); + + let mut symbols: Vec<_> = + mono_items.map(|mono_item| (mono_item, mono_item.symbol_name(tcx))).collect(); + + symbols.sort_by_key(|sym| sym.1); + + for &[(mono_item1, ref sym1), (mono_item2, ref sym2)] in symbols.array_windows() { + if sym1 == sym2 { + let span1 = mono_item1.local_span(tcx); + let span2 = mono_item2.local_span(tcx); + + // Deterministically select one of the spans for error reporting + let span = match (span1, span2) { + (Some(span1), Some(span2)) => { + Some(if span1.lo().0 > span2.lo().0 { span1 } else { span2 }) + } + (span1, span2) => span1.or(span2), + }; + + tcx.sess.emit_fatal(SymbolAlreadyDefined { span, symbol: sym1.to_string() }); + } + } +} + +fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[CodegenUnit<'_>]) { + let collection_mode = match tcx.sess.opts.unstable_opts.print_mono_items { + Some(ref s) => { + let mode = s.to_lowercase(); + let mode = mode.trim(); + if mode == "eager" { + MonoItemCollectionMode::Eager + } else { + if mode != "lazy" { + tcx.sess.emit_warning(UnknownCguCollectionMode { mode }); + } + + MonoItemCollectionMode::Lazy + } + } + None => { + if tcx.sess.link_dead_code() { + MonoItemCollectionMode::Eager + } else { + MonoItemCollectionMode::Lazy + } + } + }; + + let (items, inlining_map) = collector::collect_crate_mono_items(tcx, collection_mode); + + tcx.sess.abort_if_errors(); + + let (codegen_units, _) = tcx.sess.time("partition_and_assert_distinct_symbols", || { + sync::join( + || { + let mut codegen_units = partition( + tcx, + &mut items.iter().copied(), + tcx.sess.codegen_units(), + &inlining_map, + ); + codegen_units[0].make_primary(); + &*tcx.arena.alloc_from_iter(codegen_units) + }, + || assert_symbols_are_distinct(tcx, items.iter()), + ) + }); + + if tcx.prof.enabled() { + // Record CGU size estimates for self-profiling. + for cgu in codegen_units { + tcx.prof.artifact_size( + "codegen_unit_size_estimate", + cgu.name().as_str(), + cgu.size_estimate() as u64, + ); + } + } + + let mono_items: DefIdSet = items + .iter() + .filter_map(|mono_item| match *mono_item { + MonoItem::Fn(ref instance) => Some(instance.def_id()), + MonoItem::Static(def_id) => Some(def_id), + _ => None, + }) + .collect(); + + // Output monomorphization stats per def_id + if let SwitchWithOptPath::Enabled(ref path) = tcx.sess.opts.unstable_opts.dump_mono_stats { + if let Err(err) = + dump_mono_items_stats(tcx, &codegen_units, path, tcx.crate_name(LOCAL_CRATE)) + { + tcx.sess.emit_fatal(CouldntDumpMonoStats { error: err.to_string() }); + } + } + + if tcx.sess.opts.unstable_opts.print_mono_items.is_some() { + let mut item_to_cgus: FxHashMap<_, Vec<_>> = Default::default(); + + for cgu in codegen_units { + for (&mono_item, &linkage) in cgu.items() { + item_to_cgus.entry(mono_item).or_default().push((cgu.name(), linkage)); + } + } + + let mut item_keys: Vec<_> = items + .iter() + .map(|i| { + let mut output = with_no_trimmed_paths!(i.to_string()); + output.push_str(" @@"); + let mut empty = Vec::new(); + let cgus = item_to_cgus.get_mut(i).unwrap_or(&mut empty); + cgus.sort_by_key(|(name, _)| *name); + cgus.dedup(); + for &(ref cgu_name, (linkage, _)) in cgus.iter() { + output.push(' '); + output.push_str(cgu_name.as_str()); + + let linkage_abbrev = match linkage { + Linkage::External => "External", + Linkage::AvailableExternally => "Available", + Linkage::LinkOnceAny => "OnceAny", + Linkage::LinkOnceODR => "OnceODR", + Linkage::WeakAny => "WeakAny", + Linkage::WeakODR => "WeakODR", + Linkage::Appending => "Appending", + Linkage::Internal => "Internal", + Linkage::Private => "Private", + Linkage::ExternalWeak => "ExternalWeak", + Linkage::Common => "Common", + }; + + output.push('['); + output.push_str(linkage_abbrev); + output.push(']'); + } + output + }) + .collect(); + + item_keys.sort(); + + for item in item_keys { + println!("MONO_ITEM {item}"); + } + } + + (tcx.arena.alloc(mono_items), codegen_units) +} + +/// Outputs stats about instantiation counts and estimated size, per `MonoItem`'s +/// def, to a file in the given output directory. +fn dump_mono_items_stats<'tcx>( + tcx: TyCtxt<'tcx>, + codegen_units: &[CodegenUnit<'tcx>], + output_directory: &Option<PathBuf>, + crate_name: Symbol, +) -> Result<(), Box<dyn std::error::Error>> { + let output_directory = if let Some(ref directory) = output_directory { + fs::create_dir_all(directory)?; + directory + } else { + Path::new(".") + }; + + let format = tcx.sess.opts.unstable_opts.dump_mono_stats_format; + let ext = format.extension(); + let filename = format!("{crate_name}.mono_items.{ext}"); + let output_path = output_directory.join(&filename); + let file = File::create(&output_path)?; + let mut file = BufWriter::new(file); + + // Gather instantiated mono items grouped by def_id + let mut items_per_def_id: FxHashMap<_, Vec<_>> = Default::default(); + for cgu in codegen_units { + for (&mono_item, _) in cgu.items() { + // Avoid variable-sized compiler-generated shims + if mono_item.is_user_defined() { + items_per_def_id.entry(mono_item.def_id()).or_default().push(mono_item); + } + } + } + + #[derive(serde::Serialize)] + struct MonoItem { + name: String, + instantiation_count: usize, + size_estimate: usize, + total_estimate: usize, + } + + // Output stats sorted by total instantiated size, from heaviest to lightest + let mut stats: Vec<_> = items_per_def_id + .into_iter() + .map(|(def_id, items)| { + let name = with_no_trimmed_paths!(tcx.def_path_str(def_id)); + let instantiation_count = items.len(); + let size_estimate = items[0].size_estimate(tcx); + let total_estimate = instantiation_count * size_estimate; + MonoItem { name, instantiation_count, size_estimate, total_estimate } + }) + .collect(); + stats.sort_unstable_by_key(|item| cmp::Reverse(item.total_estimate)); + + if !stats.is_empty() { + match format { + DumpMonoStatsFormat::Json => serde_json::to_writer(file, &stats)?, + DumpMonoStatsFormat::Markdown => { + writeln!( + file, + "| Item | Instantiation count | Estimated Cost Per Instantiation | Total Estimated Cost |" + )?; + writeln!(file, "| --- | ---: | ---: | ---: |")?; + + for MonoItem { name, instantiation_count, size_estimate, total_estimate } in stats { + writeln!( + file, + "| `{name}` | {instantiation_count} | {size_estimate} | {total_estimate} |" + )?; + } + } + } + } + + Ok(()) +} + +fn codegened_and_inlined_items(tcx: TyCtxt<'_>, (): ()) -> &DefIdSet { + let (items, cgus) = tcx.collect_and_partition_mono_items(()); + let mut visited = DefIdSet::default(); + let mut result = items.clone(); + + for cgu in cgus { + for (item, _) in cgu.items() { + if let MonoItem::Fn(ref instance) = item { + let did = instance.def_id(); + if !visited.insert(did) { + continue; + } + let body = tcx.instance_mir(instance.def); + for block in body.basic_blocks.iter() { + for statement in &block.statements { + let mir::StatementKind::Coverage(_) = statement.kind else { continue }; + let scope = statement.source_info.scope; + if let Some(inlined) = scope.inlined_instance(&body.source_scopes) { + result.insert(inlined.def_id()); + } + } + } + } + } + } + + tcx.arena.alloc(result) +} + +pub fn provide(providers: &mut Providers) { + providers.collect_and_partition_mono_items = collect_and_partition_mono_items; + providers.codegened_and_inlined_items = codegened_and_inlined_items; + + providers.is_codegened_item = |tcx, def_id| { + let (all_mono_items, _) = tcx.collect_and_partition_mono_items(()); + all_mono_items.contains(&def_id) + }; + + providers.codegen_unit = |tcx, name| { + let (_, all) = tcx.collect_and_partition_mono_items(()); + all.iter() + .find(|cgu| cgu.name() == name) + .unwrap_or_else(|| panic!("failed to find cgu with name {name:?}")) + }; +} diff --git a/compiler/rustc_monomorphize/src/partitioning/default.rs b/compiler/rustc_monomorphize/src/partitioning/default.rs deleted file mode 100644 index 603b3ddc106..00000000000 --- a/compiler/rustc_monomorphize/src/partitioning/default.rs +++ /dev/null @@ -1,644 +0,0 @@ -use std::cmp; -use std::collections::hash_map::Entry; - -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_hir::def::DefKind; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_hir::definitions::DefPathDataName; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; -use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility}; -use rustc_middle::mir::mono::{InstantiationMode, MonoItem}; -use rustc_middle::ty::print::characteristic_def_id_of_type; -use rustc_middle::ty::{self, visit::TypeVisitableExt, InstanceDef, TyCtxt}; -use rustc_span::symbol::Symbol; - -use super::PartitioningCx; -use crate::collector::InliningMap; -use crate::partitioning::{MonoItemPlacement, Partition, PlacedRootMonoItems}; - -pub struct DefaultPartitioning; - -impl<'tcx> Partition<'tcx> for DefaultPartitioning { - fn place_root_mono_items<I>( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - mono_items: &mut I, - ) -> PlacedRootMonoItems<'tcx> - where - I: Iterator<Item = MonoItem<'tcx>>, - { - let mut roots = FxHashSet::default(); - let mut codegen_units = FxHashMap::default(); - let is_incremental_build = cx.tcx.sess.opts.incremental.is_some(); - let mut internalization_candidates = FxHashSet::default(); - - // Determine if monomorphizations instantiated in this crate will be made - // available to downstream crates. This depends on whether we are in - // share-generics mode and whether the current crate can even have - // downstream crates. - let export_generics = - cx.tcx.sess.opts.share_generics() && cx.tcx.local_crate_exports_generics(); - - let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx); - let cgu_name_cache = &mut FxHashMap::default(); - - for mono_item in mono_items { - match mono_item.instantiation_mode(cx.tcx) { - InstantiationMode::GloballyShared { .. } => {} - InstantiationMode::LocalCopy => continue, - } - - let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item); - let is_volatile = is_incremental_build && mono_item.is_generic_fn(); - - let codegen_unit_name = match characteristic_def_id { - Some(def_id) => compute_codegen_unit_name( - cx.tcx, - cgu_name_builder, - def_id, - is_volatile, - cgu_name_cache, - ), - None => fallback_cgu_name(cgu_name_builder), - }; - - let codegen_unit = codegen_units - .entry(codegen_unit_name) - .or_insert_with(|| CodegenUnit::new(codegen_unit_name)); - - let mut can_be_internalized = true; - let (linkage, visibility) = mono_item_linkage_and_visibility( - cx.tcx, - &mono_item, - &mut can_be_internalized, - export_generics, - ); - if visibility == Visibility::Hidden && can_be_internalized { - internalization_candidates.insert(mono_item); - } - - codegen_unit.items_mut().insert(mono_item, (linkage, visibility)); - roots.insert(mono_item); - } - - // Always ensure we have at least one CGU; otherwise, if we have a - // crate with just types (for example), we could wind up with no CGU. - if codegen_units.is_empty() { - let codegen_unit_name = fallback_cgu_name(cgu_name_builder); - codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name)); - } - - let codegen_units = codegen_units.into_values().collect(); - PlacedRootMonoItems { codegen_units, roots, internalization_candidates } - } - - fn merge_codegen_units( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut Vec<CodegenUnit<'tcx>>, - ) { - assert!(cx.target_cgu_count >= 1); - - // Note that at this point in time the `codegen_units` here may not be - // in a deterministic order (but we know they're deterministically the - // same set). We want this merging to produce a deterministic ordering - // of codegen units from the input. - // - // Due to basically how we've implemented the merging below (merge the - // two smallest into each other) we're sure to start off with a - // deterministic order (sorted by name). This'll mean that if two cgus - // have the same size the stable sort below will keep everything nice - // and deterministic. - codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); - - // This map keeps track of what got merged into what. - let mut cgu_contents: FxHashMap<Symbol, Vec<Symbol>> = - codegen_units.iter().map(|cgu| (cgu.name(), vec![cgu.name()])).collect(); - - // Merge the two smallest codegen units until the target size is - // reached. - while codegen_units.len() > cx.target_cgu_count { - // Sort small cgus to the back - codegen_units.sort_by_cached_key(|cgu| cmp::Reverse(cgu.size_estimate())); - let mut smallest = codegen_units.pop().unwrap(); - let second_smallest = codegen_units.last_mut().unwrap(); - - // Move the mono-items from `smallest` to `second_smallest` - second_smallest.modify_size_estimate(smallest.size_estimate()); - for (k, v) in smallest.items_mut().drain() { - second_smallest.items_mut().insert(k, v); - } - - // Record that `second_smallest` now contains all the stuff that was - // in `smallest` before. - let mut consumed_cgu_names = cgu_contents.remove(&smallest.name()).unwrap(); - cgu_contents.get_mut(&second_smallest.name()).unwrap().append(&mut consumed_cgu_names); - - debug!( - "CodegenUnit {} merged into CodegenUnit {}", - smallest.name(), - second_smallest.name() - ); - } - - let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx); - - if cx.tcx.sess.opts.incremental.is_some() { - // If we are doing incremental compilation, we want CGU names to - // reflect the path of the source level module they correspond to. - // For CGUs that contain the code of multiple modules because of the - // merging done above, we use a concatenation of the names of all - // contained CGUs. - let new_cgu_names: FxHashMap<Symbol, String> = cgu_contents - .into_iter() - // This `filter` makes sure we only update the name of CGUs that - // were actually modified by merging. - .filter(|(_, cgu_contents)| cgu_contents.len() > 1) - .map(|(current_cgu_name, cgu_contents)| { - let mut cgu_contents: Vec<&str> = - cgu_contents.iter().map(|s| s.as_str()).collect(); - - // Sort the names, so things are deterministic and easy to - // predict. We are sorting primitive `&str`s here so we can - // use unstable sort. - cgu_contents.sort_unstable(); - - (current_cgu_name, cgu_contents.join("--")) - }) - .collect(); - - for cgu in codegen_units.iter_mut() { - if let Some(new_cgu_name) = new_cgu_names.get(&cgu.name()) { - if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names { - cgu.set_name(Symbol::intern(&new_cgu_name)); - } else { - // If we don't require CGU names to be human-readable, - // we use a fixed length hash of the composite CGU name - // instead. - let new_cgu_name = CodegenUnit::mangle_name(&new_cgu_name); - cgu.set_name(Symbol::intern(&new_cgu_name)); - } - } - } - } else { - // If we are compiling non-incrementally we just generate simple CGU - // names containing an index. - for (index, cgu) in codegen_units.iter_mut().enumerate() { - let numbered_codegen_unit_name = - cgu_name_builder.build_cgu_name_no_mangle(LOCAL_CRATE, &["cgu"], Some(index)); - cgu.set_name(numbered_codegen_unit_name); - } - } - } - - fn place_inlined_mono_items( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - roots: FxHashSet<MonoItem<'tcx>>, - ) -> FxHashMap<MonoItem<'tcx>, MonoItemPlacement> { - let mut mono_item_placements = FxHashMap::default(); - - let single_codegen_unit = codegen_units.len() == 1; - - for old_codegen_unit in codegen_units.iter_mut() { - // Collect all items that need to be available in this codegen unit. - let mut reachable = FxHashSet::default(); - for root in old_codegen_unit.items().keys() { - follow_inlining(*root, cx.inlining_map, &mut reachable); - } - - let mut new_codegen_unit = CodegenUnit::new(old_codegen_unit.name()); - - // Add all monomorphizations that are not already there. - for mono_item in reachable { - if let Some(linkage) = old_codegen_unit.items().get(&mono_item) { - // This is a root, just copy it over. - new_codegen_unit.items_mut().insert(mono_item, *linkage); - } else { - if roots.contains(&mono_item) { - bug!( - "GloballyShared mono-item inlined into other CGU: \ - {:?}", - mono_item - ); - } - - // This is a CGU-private copy. - new_codegen_unit - .items_mut() - .insert(mono_item, (Linkage::Internal, Visibility::Default)); - } - - if !single_codegen_unit { - // If there is more than one codegen unit, we need to keep track - // in which codegen units each monomorphization is placed. - match mono_item_placements.entry(mono_item) { - Entry::Occupied(e) => { - let placement = e.into_mut(); - debug_assert!(match *placement { - MonoItemPlacement::SingleCgu { cgu_name } => { - cgu_name != new_codegen_unit.name() - } - MonoItemPlacement::MultipleCgus => true, - }); - *placement = MonoItemPlacement::MultipleCgus; - } - Entry::Vacant(e) => { - e.insert(MonoItemPlacement::SingleCgu { - cgu_name: new_codegen_unit.name(), - }); - } - } - } - } - - *old_codegen_unit = new_codegen_unit; - } - - return mono_item_placements; - - fn follow_inlining<'tcx>( - mono_item: MonoItem<'tcx>, - inlining_map: &InliningMap<'tcx>, - visited: &mut FxHashSet<MonoItem<'tcx>>, - ) { - if !visited.insert(mono_item) { - return; - } - - inlining_map.with_inlining_candidates(mono_item, |target| { - follow_inlining(target, inlining_map, visited); - }); - } - } - - fn internalize_symbols( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - mono_item_placements: FxHashMap<MonoItem<'tcx>, MonoItemPlacement>, - internalization_candidates: FxHashSet<MonoItem<'tcx>>, - ) { - if codegen_units.len() == 1 { - // Fast path for when there is only one codegen unit. In this case we - // can internalize all candidates, since there is nowhere else they - // could be accessed from. - for cgu in codegen_units { - for candidate in &internalization_candidates { - cgu.items_mut().insert(*candidate, (Linkage::Internal, Visibility::Default)); - } - } - - return; - } - - // Build a map from every monomorphization to all the monomorphizations that - // reference it. - let mut accessor_map: FxHashMap<MonoItem<'tcx>, Vec<MonoItem<'tcx>>> = Default::default(); - cx.inlining_map.iter_accesses(|accessor, accessees| { - for accessee in accessees { - accessor_map.entry(*accessee).or_default().push(accessor); - } - }); - - // For each internalization candidates in each codegen unit, check if it is - // accessed from outside its defining codegen unit. - for cgu in codegen_units { - let home_cgu = MonoItemPlacement::SingleCgu { cgu_name: cgu.name() }; - - for (accessee, linkage_and_visibility) in cgu.items_mut() { - if !internalization_candidates.contains(accessee) { - // This item is no candidate for internalizing, so skip it. - continue; - } - debug_assert_eq!(mono_item_placements[accessee], home_cgu); - - if let Some(accessors) = accessor_map.get(accessee) { - if accessors - .iter() - .filter_map(|accessor| { - // Some accessors might not have been - // instantiated. We can safely ignore those. - mono_item_placements.get(accessor) - }) - .any(|placement| *placement != home_cgu) - { - // Found an accessor from another CGU, so skip to the next - // item without marking this one as internal. - continue; - } - } - - // If we got here, we did not find any accesses from other CGUs, - // so it's fine to make this monomorphization internal. - *linkage_and_visibility = (Linkage::Internal, Visibility::Default); - } - } - } -} - -fn characteristic_def_id_of_mono_item<'tcx>( - tcx: TyCtxt<'tcx>, - mono_item: MonoItem<'tcx>, -) -> Option<DefId> { - match mono_item { - MonoItem::Fn(instance) => { - let def_id = match instance.def { - ty::InstanceDef::Item(def) => def, - ty::InstanceDef::VTableShim(..) - | ty::InstanceDef::ReifyShim(..) - | ty::InstanceDef::FnPtrShim(..) - | ty::InstanceDef::ClosureOnceShim { .. } - | ty::InstanceDef::Intrinsic(..) - | ty::InstanceDef::DropGlue(..) - | ty::InstanceDef::Virtual(..) - | ty::InstanceDef::CloneShim(..) - | ty::InstanceDef::ThreadLocalShim(..) - | ty::InstanceDef::FnPtrAddrShim(..) => return None, - }; - - // If this is a method, we want to put it into the same module as - // its self-type. If the self-type does not provide a characteristic - // DefId, we use the location of the impl after all. - - if tcx.trait_of_item(def_id).is_some() { - let self_ty = instance.substs.type_at(0); - // This is a default implementation of a trait method. - return characteristic_def_id_of_type(self_ty).or(Some(def_id)); - } - - if let Some(impl_def_id) = tcx.impl_of_method(def_id) { - if tcx.sess.opts.incremental.is_some() - && tcx.trait_id_of_impl(impl_def_id) == tcx.lang_items().drop_trait() - { - // Put `Drop::drop` into the same cgu as `drop_in_place` - // since `drop_in_place` is the only thing that can - // call it. - return None; - } - - // When polymorphization is enabled, methods which do not depend on their generic - // parameters, but the self-type of their impl block do will fail to normalize. - if !tcx.sess.opts.unstable_opts.polymorphize || !instance.has_param() { - // This is a method within an impl, find out what the self-type is: - let impl_self_ty = tcx.subst_and_normalize_erasing_regions( - instance.substs, - ty::ParamEnv::reveal_all(), - tcx.type_of(impl_def_id), - ); - if let Some(def_id) = characteristic_def_id_of_type(impl_self_ty) { - return Some(def_id); - } - } - } - - Some(def_id) - } - MonoItem::Static(def_id) => Some(def_id), - MonoItem::GlobalAsm(item_id) => Some(item_id.owner_id.to_def_id()), - } -} - -fn compute_codegen_unit_name( - tcx: TyCtxt<'_>, - name_builder: &mut CodegenUnitNameBuilder<'_>, - def_id: DefId, - volatile: bool, - cache: &mut CguNameCache, -) -> Symbol { - // Find the innermost module that is not nested within a function. - let mut current_def_id = def_id; - let mut cgu_def_id = None; - // Walk backwards from the item we want to find the module for. - loop { - if current_def_id.is_crate_root() { - if cgu_def_id.is_none() { - // If we have not found a module yet, take the crate root. - cgu_def_id = Some(def_id.krate.as_def_id()); - } - break; - } else if tcx.def_kind(current_def_id) == DefKind::Mod { - if cgu_def_id.is_none() { - cgu_def_id = Some(current_def_id); - } - } else { - // If we encounter something that is not a module, throw away - // any module that we've found so far because we now know that - // it is nested within something else. - cgu_def_id = None; - } - - current_def_id = tcx.parent(current_def_id); - } - - let cgu_def_id = cgu_def_id.unwrap(); - - *cache.entry((cgu_def_id, volatile)).or_insert_with(|| { - let def_path = tcx.def_path(cgu_def_id); - - let components = def_path.data.iter().map(|part| match part.data.name() { - DefPathDataName::Named(name) => name, - DefPathDataName::Anon { .. } => unreachable!(), - }); - - let volatile_suffix = volatile.then_some("volatile"); - - name_builder.build_cgu_name(def_path.krate, components, volatile_suffix) - }) -} - -// Anything we can't find a proper codegen unit for goes into this. -fn fallback_cgu_name(name_builder: &mut CodegenUnitNameBuilder<'_>) -> Symbol { - name_builder.build_cgu_name(LOCAL_CRATE, &["fallback"], Some("cgu")) -} - -fn mono_item_linkage_and_visibility<'tcx>( - tcx: TyCtxt<'tcx>, - mono_item: &MonoItem<'tcx>, - can_be_internalized: &mut bool, - export_generics: bool, -) -> (Linkage, Visibility) { - if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) { - return (explicit_linkage, Visibility::Default); - } - let vis = mono_item_visibility(tcx, mono_item, can_be_internalized, export_generics); - (Linkage::External, vis) -} - -type CguNameCache = FxHashMap<(DefId, bool), Symbol>; - -fn static_visibility<'tcx>( - tcx: TyCtxt<'tcx>, - can_be_internalized: &mut bool, - def_id: DefId, -) -> Visibility { - if tcx.is_reachable_non_generic(def_id) { - *can_be_internalized = false; - default_visibility(tcx, def_id, false) - } else { - Visibility::Hidden - } -} - -fn mono_item_visibility<'tcx>( - tcx: TyCtxt<'tcx>, - mono_item: &MonoItem<'tcx>, - can_be_internalized: &mut bool, - export_generics: bool, -) -> Visibility { - let instance = match mono_item { - // This is pretty complicated; see below. - MonoItem::Fn(instance) => instance, - - // Misc handling for generics and such, but otherwise: - MonoItem::Static(def_id) => return static_visibility(tcx, can_be_internalized, *def_id), - MonoItem::GlobalAsm(item_id) => { - return static_visibility(tcx, can_be_internalized, item_id.owner_id.to_def_id()); - } - }; - - let def_id = match instance.def { - InstanceDef::Item(def_id) | InstanceDef::DropGlue(def_id, Some(_)) => def_id, - - // We match the visibility of statics here - InstanceDef::ThreadLocalShim(def_id) => { - return static_visibility(tcx, can_be_internalized, def_id); - } - - // These are all compiler glue and such, never exported, always hidden. - InstanceDef::VTableShim(..) - | InstanceDef::ReifyShim(..) - | InstanceDef::FnPtrShim(..) - | InstanceDef::Virtual(..) - | InstanceDef::Intrinsic(..) - | InstanceDef::ClosureOnceShim { .. } - | InstanceDef::DropGlue(..) - | InstanceDef::CloneShim(..) - | InstanceDef::FnPtrAddrShim(..) => return Visibility::Hidden, - }; - - // The `start_fn` lang item is actually a monomorphized instance of a - // function in the standard library, used for the `main` function. We don't - // want to export it so we tag it with `Hidden` visibility but this symbol - // is only referenced from the actual `main` symbol which we unfortunately - // don't know anything about during partitioning/collection. As a result we - // forcibly keep this symbol out of the `internalization_candidates` set. - // - // FIXME: eventually we don't want to always force this symbol to have - // hidden visibility, it should indeed be a candidate for - // internalization, but we have to understand that it's referenced - // from the `main` symbol we'll generate later. - // - // This may be fixable with a new `InstanceDef` perhaps? Unsure! - if tcx.lang_items().start_fn() == Some(def_id) { - *can_be_internalized = false; - return Visibility::Hidden; - } - - let is_generic = instance.substs.non_erasable_generics().next().is_some(); - - // Upstream `DefId` instances get different handling than local ones. - let Some(def_id) = def_id.as_local() else { - return if export_generics && is_generic { - // If it is an upstream monomorphization and we export generics, we must make - // it available to downstream crates. - *can_be_internalized = false; - default_visibility(tcx, def_id, true) - } else { - Visibility::Hidden - }; - }; - - if is_generic { - if export_generics { - if tcx.is_unreachable_local_definition(def_id) { - // This instance cannot be used from another crate. - Visibility::Hidden - } else { - // This instance might be useful in a downstream crate. - *can_be_internalized = false; - default_visibility(tcx, def_id.to_def_id(), true) - } - } else { - // We are not exporting generics or the definition is not reachable - // for downstream crates, we can internalize its instantiations. - Visibility::Hidden - } - } else { - // If this isn't a generic function then we mark this a `Default` if - // this is a reachable item, meaning that it's a symbol other crates may - // access when they link to us. - if tcx.is_reachable_non_generic(def_id.to_def_id()) { - *can_be_internalized = false; - debug_assert!(!is_generic); - return default_visibility(tcx, def_id.to_def_id(), false); - } - - // If this isn't reachable then we're gonna tag this with `Hidden` - // visibility. In some situations though we'll want to prevent this - // symbol from being internalized. - // - // There's two categories of items here: - // - // * First is weak lang items. These are basically mechanisms for - // libcore to forward-reference symbols defined later in crates like - // the standard library or `#[panic_handler]` definitions. The - // definition of these weak lang items needs to be referencable by - // libcore, so we're no longer a candidate for internalization. - // Removal of these functions can't be done by LLVM but rather must be - // done by the linker as it's a non-local decision. - // - // * Second is "std internal symbols". Currently this is primarily used - // for allocator symbols. Allocators are a little weird in their - // implementation, but the idea is that the compiler, at the last - // minute, defines an allocator with an injected object file. The - // `alloc` crate references these symbols (`__rust_alloc`) and the - // definition doesn't get hooked up until a linked crate artifact is - // generated. - // - // The symbols synthesized by the compiler (`__rust_alloc`) are thin - // veneers around the actual implementation, some other symbol which - // implements the same ABI. These symbols (things like `__rg_alloc`, - // `__rdl_alloc`, `__rde_alloc`, etc), are all tagged with "std - // internal symbols". - // - // The std-internal symbols here **should not show up in a dll as an - // exported interface**, so they return `false` from - // `is_reachable_non_generic` above and we'll give them `Hidden` - // visibility below. Like the weak lang items, though, we can't let - // LLVM internalize them as this decision is left up to the linker to - // omit them, so prevent them from being internalized. - let attrs = tcx.codegen_fn_attrs(def_id); - if attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) { - *can_be_internalized = false; - } - - Visibility::Hidden - } -} - -fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibility { - if !tcx.sess.target.default_hidden_visibility { - return Visibility::Default; - } - - // Generic functions never have export-level C. - if is_generic { - return Visibility::Hidden; - } - - // Things with export level C don't get instantiated in - // downstream crates. - if !id.is_local() { - return Visibility::Hidden; - } - - // C-export level items remain at `Default`, all other internal - // items become `Hidden`. - match tcx.reachable_non_generics(id.krate).get(&id) { - Some(SymbolExportInfo { level: SymbolExportLevel::C, .. }) => Visibility::Default, - _ => Visibility::Hidden, - } -} diff --git a/compiler/rustc_monomorphize/src/partitioning/mod.rs b/compiler/rustc_monomorphize/src/partitioning/mod.rs deleted file mode 100644 index d0b23ca9ea4..00000000000 --- a/compiler/rustc_monomorphize/src/partitioning/mod.rs +++ /dev/null @@ -1,673 +0,0 @@ -//! Partitioning Codegen Units for Incremental Compilation -//! ====================================================== -//! -//! The task of this module is to take the complete set of monomorphizations of -//! a crate and produce a set of codegen units from it, where a codegen unit -//! is a named set of (mono-item, linkage) pairs. That is, this module -//! decides which monomorphization appears in which codegen units with which -//! linkage. The following paragraphs describe some of the background on the -//! partitioning scheme. -//! -//! The most important opportunity for saving on compilation time with -//! incremental compilation is to avoid re-codegenning and re-optimizing code. -//! Since the unit of codegen and optimization for LLVM is "modules" or, how -//! we call them "codegen units", the particulars of how much time can be saved -//! by incremental compilation are tightly linked to how the output program is -//! partitioned into these codegen units prior to passing it to LLVM -- -//! especially because we have to treat codegen units as opaque entities once -//! they are created: There is no way for us to incrementally update an existing -//! LLVM module and so we have to build any such module from scratch if it was -//! affected by some change in the source code. -//! -//! From that point of view it would make sense to maximize the number of -//! codegen units by, for example, putting each function into its own module. -//! That way only those modules would have to be re-compiled that were actually -//! affected by some change, minimizing the number of functions that could have -//! been re-used but just happened to be located in a module that is -//! re-compiled. -//! -//! However, since LLVM optimization does not work across module boundaries, -//! using such a highly granular partitioning would lead to very slow runtime -//! code since it would effectively prohibit inlining and other inter-procedure -//! optimizations. We want to avoid that as much as possible. -//! -//! Thus we end up with a trade-off: The bigger the codegen units, the better -//! LLVM's optimizer can do its work, but also the smaller the compilation time -//! reduction we get from incremental compilation. -//! -//! Ideally, we would create a partitioning such that there are few big codegen -//! units with few interdependencies between them. For now though, we use the -//! following heuristic to determine the partitioning: -//! -//! - There are two codegen units for every source-level module: -//! - One for "stable", that is non-generic, code -//! - One for more "volatile" code, i.e., monomorphized instances of functions -//! defined in that module -//! -//! In order to see why this heuristic makes sense, let's take a look at when a -//! codegen unit can get invalidated: -//! -//! 1. The most straightforward case is when the BODY of a function or global -//! changes. Then any codegen unit containing the code for that item has to be -//! re-compiled. Note that this includes all codegen units where the function -//! has been inlined. -//! -//! 2. The next case is when the SIGNATURE of a function or global changes. In -//! this case, all codegen units containing a REFERENCE to that item have to be -//! re-compiled. This is a superset of case 1. -//! -//! 3. The final and most subtle case is when a REFERENCE to a generic function -//! is added or removed somewhere. Even though the definition of the function -//! might be unchanged, a new REFERENCE might introduce a new monomorphized -//! instance of this function which has to be placed and compiled somewhere. -//! Conversely, when removing a REFERENCE, it might have been the last one with -//! that particular set of generic arguments and thus we have to remove it. -//! -//! From the above we see that just using one codegen unit per source-level -//! module is not such a good idea, since just adding a REFERENCE to some -//! generic item somewhere else would invalidate everything within the module -//! containing the generic item. The heuristic above reduces this detrimental -//! side-effect of references a little by at least not touching the non-generic -//! code of the module. -//! -//! A Note on Inlining -//! ------------------ -//! As briefly mentioned above, in order for LLVM to be able to inline a -//! function call, the body of the function has to be available in the LLVM -//! module where the call is made. This has a few consequences for partitioning: -//! -//! - The partitioning algorithm has to take care of placing functions into all -//! codegen units where they should be available for inlining. It also has to -//! decide on the correct linkage for these functions. -//! -//! - The partitioning algorithm has to know which functions are likely to get -//! inlined, so it can distribute function instantiations accordingly. Since -//! there is no way of knowing for sure which functions LLVM will decide to -//! inline in the end, we apply a heuristic here: Only functions marked with -//! `#[inline]` are considered for inlining by the partitioner. The current -//! implementation will not try to determine if a function is likely to be -//! inlined by looking at the functions definition. -//! -//! Note though that as a side-effect of creating a codegen units per -//! source-level module, functions from the same module will be available for -//! inlining, even when they are not marked `#[inline]`. - -mod default; - -use std::cmp; -use std::fs::{self, File}; -use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; - -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_data_structures::sync; -use rustc_hir::def_id::{DefIdSet, LOCAL_CRATE}; -use rustc_middle::mir; -use rustc_middle::mir::mono::MonoItem; -use rustc_middle::mir::mono::{CodegenUnit, Linkage}; -use rustc_middle::query::Providers; -use rustc_middle::ty::print::with_no_trimmed_paths; -use rustc_middle::ty::TyCtxt; -use rustc_session::config::{DumpMonoStatsFormat, SwitchWithOptPath}; -use rustc_span::symbol::Symbol; - -use crate::collector::InliningMap; -use crate::collector::{self, MonoItemCollectionMode}; -use crate::errors::{ - CouldntDumpMonoStats, SymbolAlreadyDefined, UnknownCguCollectionMode, UnknownPartitionStrategy, -}; - -enum Partitioner { - Default(default::DefaultPartitioning), - // Other partitioning strategies can go here. - Unknown, -} - -impl<'tcx> Partition<'tcx> for Partitioner { - fn place_root_mono_items<I>( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - mono_items: &mut I, - ) -> PlacedRootMonoItems<'tcx> - where - I: Iterator<Item = MonoItem<'tcx>>, - { - match self { - Partitioner::Default(partitioner) => partitioner.place_root_mono_items(cx, mono_items), - Partitioner::Unknown => cx.tcx.sess.emit_fatal(UnknownPartitionStrategy), - } - } - - fn merge_codegen_units( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut Vec<CodegenUnit<'tcx>>, - ) { - match self { - Partitioner::Default(partitioner) => partitioner.merge_codegen_units(cx, codegen_units), - Partitioner::Unknown => cx.tcx.sess.emit_fatal(UnknownPartitionStrategy), - } - } - - fn place_inlined_mono_items( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - roots: FxHashSet<MonoItem<'tcx>>, - ) -> FxHashMap<MonoItem<'tcx>, MonoItemPlacement> { - match self { - Partitioner::Default(partitioner) => { - partitioner.place_inlined_mono_items(cx, codegen_units, roots) - } - Partitioner::Unknown => cx.tcx.sess.emit_fatal(UnknownPartitionStrategy), - } - } - - fn internalize_symbols( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - mono_item_placements: FxHashMap<MonoItem<'tcx>, MonoItemPlacement>, - internalization_candidates: FxHashSet<MonoItem<'tcx>>, - ) { - match self { - Partitioner::Default(partitioner) => partitioner.internalize_symbols( - cx, - codegen_units, - mono_item_placements, - internalization_candidates, - ), - Partitioner::Unknown => cx.tcx.sess.emit_fatal(UnknownPartitionStrategy), - } - } -} - -struct PartitioningCx<'a, 'tcx> { - tcx: TyCtxt<'tcx>, - target_cgu_count: usize, - inlining_map: &'a InliningMap<'tcx>, -} - -pub struct PlacedRootMonoItems<'tcx> { - codegen_units: Vec<CodegenUnit<'tcx>>, - roots: FxHashSet<MonoItem<'tcx>>, - internalization_candidates: FxHashSet<MonoItem<'tcx>>, -} - -trait Partition<'tcx> { - fn place_root_mono_items<I>( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - mono_items: &mut I, - ) -> PlacedRootMonoItems<'tcx> - where - I: Iterator<Item = MonoItem<'tcx>>; - - fn merge_codegen_units( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut Vec<CodegenUnit<'tcx>>, - ); - - fn place_inlined_mono_items( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - roots: FxHashSet<MonoItem<'tcx>>, - ) -> FxHashMap<MonoItem<'tcx>, MonoItemPlacement>; - - fn internalize_symbols( - &mut self, - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], - mono_item_placements: FxHashMap<MonoItem<'tcx>, MonoItemPlacement>, - internalization_candidates: FxHashSet<MonoItem<'tcx>>, - ); -} - -fn get_partitioner(tcx: TyCtxt<'_>) -> Partitioner { - let strategy = match &tcx.sess.opts.unstable_opts.cgu_partitioning_strategy { - None => "default", - Some(s) => &s[..], - }; - - match strategy { - "default" => Partitioner::Default(default::DefaultPartitioning), - _ => Partitioner::Unknown, - } -} - -fn partition<'tcx, I>( - tcx: TyCtxt<'tcx>, - mono_items: &mut I, - max_cgu_count: usize, - inlining_map: &InliningMap<'tcx>, -) -> Vec<CodegenUnit<'tcx>> -where - I: Iterator<Item = MonoItem<'tcx>>, -{ - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning"); - - let mut partitioner = get_partitioner(tcx); - let cx = &PartitioningCx { tcx, target_cgu_count: max_cgu_count, inlining_map }; - // In the first step, we place all regular monomorphizations into their - // respective 'home' codegen unit. Regular monomorphizations are all - // functions and statics defined in the local crate. - let PlacedRootMonoItems { mut codegen_units, roots, internalization_candidates } = { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_roots"); - partitioner.place_root_mono_items(cx, mono_items) - }; - - for cgu in &mut codegen_units { - cgu.create_size_estimate(tcx); - } - - debug_dump(tcx, "INITIAL PARTITIONING", &codegen_units); - - // Merge until we have at most `max_cgu_count` codegen units. - // `merge_codegen_units` is responsible for updating the CGU size - // estimates. - { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_merge_cgus"); - partitioner.merge_codegen_units(cx, &mut codegen_units); - debug_dump(tcx, "POST MERGING", &codegen_units); - } - - // In the next step, we use the inlining map to determine which additional - // monomorphizations have to go into each codegen unit. These additional - // monomorphizations can be drop-glue, functions from external crates, and - // local functions the definition of which is marked with `#[inline]`. - let mono_item_placements = { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_inline_items"); - partitioner.place_inlined_mono_items(cx, &mut codegen_units, roots) - }; - - for cgu in &mut codegen_units { - cgu.create_size_estimate(tcx); - } - - debug_dump(tcx, "POST INLINING", &codegen_units); - - // Next we try to make as many symbols "internal" as possible, so LLVM has - // more freedom to optimize. - if !tcx.sess.link_dead_code() { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols"); - partitioner.internalize_symbols( - cx, - &mut codegen_units, - mono_item_placements, - internalization_candidates, - ); - } - - let instrument_dead_code = - tcx.sess.instrument_coverage() && !tcx.sess.instrument_coverage_except_unused_functions(); - - if instrument_dead_code { - assert!( - codegen_units.len() > 0, - "There must be at least one CGU that code coverage data can be generated in." - ); - - // Find the smallest CGU that has exported symbols and put the dead - // function stubs in that CGU. We look for exported symbols to increase - // the likelihood the linker won't throw away the dead functions. - // FIXME(#92165): In order to truly resolve this, we need to make sure - // the object file (CGU) containing the dead function stubs is included - // in the final binary. This will probably require forcing these - // function symbols to be included via `-u` or `/include` linker args. - let mut cgus: Vec<_> = codegen_units.iter_mut().collect(); - cgus.sort_by_key(|cgu| cgu.size_estimate()); - - let dead_code_cgu = - if let Some(cgu) = cgus.into_iter().rev().find(|cgu| { - cgu.items().iter().any(|(_, (linkage, _))| *linkage == Linkage::External) - }) { - cgu - } else { - // If there are no CGUs that have externally linked items, - // then we just pick the first CGU as a fallback. - &mut codegen_units[0] - }; - dead_code_cgu.make_code_coverage_dead_code_cgu(); - } - - // Finally, sort by codegen unit name, so that we get deterministic results. - codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); - - debug_dump(tcx, "FINAL", &codegen_units); - - codegen_units -} - -/// For symbol internalization, we need to know whether a symbol/mono-item is -/// accessed from outside the codegen unit it is defined in. This type is used -/// to keep track of that. -#[derive(Clone, PartialEq, Eq, Debug)] -enum MonoItemPlacement { - SingleCgu { cgu_name: Symbol }, - MultipleCgus, -} - -fn debug_dump<'a, 'tcx: 'a>(tcx: TyCtxt<'tcx>, label: &str, cgus: &[CodegenUnit<'tcx>]) { - let dump = move || { - use std::fmt::Write; - - let num_cgus = cgus.len(); - let max = cgus.iter().map(|cgu| cgu.size_estimate()).max().unwrap(); - let min = cgus.iter().map(|cgu| cgu.size_estimate()).min().unwrap(); - let ratio = max as f64 / min as f64; - - let s = &mut String::new(); - let _ = writeln!( - s, - "{label} ({num_cgus} CodegenUnits, max={max}, min={min}, max/min={ratio:.1}):" - ); - for cgu in cgus { - let _ = - writeln!(s, "CodegenUnit {} estimated size {}:", cgu.name(), cgu.size_estimate()); - - for (mono_item, linkage) in cgu.items() { - let symbol_name = mono_item.symbol_name(tcx).name; - let symbol_hash_start = symbol_name.rfind('h'); - let symbol_hash = symbol_hash_start.map_or("<no hash>", |i| &symbol_name[i..]); - - let _ = with_no_trimmed_paths!(writeln!( - s, - " - {} [{:?}] [{}] estimated size {}", - mono_item, - linkage, - symbol_hash, - mono_item.size_estimate(tcx) - )); - } - - let _ = writeln!(s); - } - - std::mem::take(s) - }; - - debug!("{}", dump()); -} - -#[inline(never)] // give this a place in the profiler -fn assert_symbols_are_distinct<'a, 'tcx, I>(tcx: TyCtxt<'tcx>, mono_items: I) -where - I: Iterator<Item = &'a MonoItem<'tcx>>, - 'tcx: 'a, -{ - let _prof_timer = tcx.prof.generic_activity("assert_symbols_are_distinct"); - - let mut symbols: Vec<_> = - mono_items.map(|mono_item| (mono_item, mono_item.symbol_name(tcx))).collect(); - - symbols.sort_by_key(|sym| sym.1); - - for &[(mono_item1, ref sym1), (mono_item2, ref sym2)] in symbols.array_windows() { - if sym1 == sym2 { - let span1 = mono_item1.local_span(tcx); - let span2 = mono_item2.local_span(tcx); - - // Deterministically select one of the spans for error reporting - let span = match (span1, span2) { - (Some(span1), Some(span2)) => { - Some(if span1.lo().0 > span2.lo().0 { span1 } else { span2 }) - } - (span1, span2) => span1.or(span2), - }; - - tcx.sess.emit_fatal(SymbolAlreadyDefined { span, symbol: sym1.to_string() }); - } - } -} - -fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[CodegenUnit<'_>]) { - let collection_mode = match tcx.sess.opts.unstable_opts.print_mono_items { - Some(ref s) => { - let mode = s.to_lowercase(); - let mode = mode.trim(); - if mode == "eager" { - MonoItemCollectionMode::Eager - } else { - if mode != "lazy" { - tcx.sess.emit_warning(UnknownCguCollectionMode { mode }); - } - - MonoItemCollectionMode::Lazy - } - } - None => { - if tcx.sess.link_dead_code() { - MonoItemCollectionMode::Eager - } else { - MonoItemCollectionMode::Lazy - } - } - }; - - let (items, inlining_map) = collector::collect_crate_mono_items(tcx, collection_mode); - - tcx.sess.abort_if_errors(); - - let (codegen_units, _) = tcx.sess.time("partition_and_assert_distinct_symbols", || { - sync::join( - || { - let mut codegen_units = partition( - tcx, - &mut items.iter().copied(), - tcx.sess.codegen_units(), - &inlining_map, - ); - codegen_units[0].make_primary(); - &*tcx.arena.alloc_from_iter(codegen_units) - }, - || assert_symbols_are_distinct(tcx, items.iter()), - ) - }); - - if tcx.prof.enabled() { - // Record CGU size estimates for self-profiling. - for cgu in codegen_units { - tcx.prof.artifact_size( - "codegen_unit_size_estimate", - cgu.name().as_str(), - cgu.size_estimate() as u64, - ); - } - } - - let mono_items: DefIdSet = items - .iter() - .filter_map(|mono_item| match *mono_item { - MonoItem::Fn(ref instance) => Some(instance.def_id()), - MonoItem::Static(def_id) => Some(def_id), - _ => None, - }) - .collect(); - - // Output monomorphization stats per def_id - if let SwitchWithOptPath::Enabled(ref path) = tcx.sess.opts.unstable_opts.dump_mono_stats { - if let Err(err) = - dump_mono_items_stats(tcx, &codegen_units, path, tcx.crate_name(LOCAL_CRATE)) - { - tcx.sess.emit_fatal(CouldntDumpMonoStats { error: err.to_string() }); - } - } - - if tcx.sess.opts.unstable_opts.print_mono_items.is_some() { - let mut item_to_cgus: FxHashMap<_, Vec<_>> = Default::default(); - - for cgu in codegen_units { - for (&mono_item, &linkage) in cgu.items() { - item_to_cgus.entry(mono_item).or_default().push((cgu.name(), linkage)); - } - } - - let mut item_keys: Vec<_> = items - .iter() - .map(|i| { - let mut output = with_no_trimmed_paths!(i.to_string()); - output.push_str(" @@"); - let mut empty = Vec::new(); - let cgus = item_to_cgus.get_mut(i).unwrap_or(&mut empty); - cgus.sort_by_key(|(name, _)| *name); - cgus.dedup(); - for &(ref cgu_name, (linkage, _)) in cgus.iter() { - output.push(' '); - output.push_str(cgu_name.as_str()); - - let linkage_abbrev = match linkage { - Linkage::External => "External", - Linkage::AvailableExternally => "Available", - Linkage::LinkOnceAny => "OnceAny", - Linkage::LinkOnceODR => "OnceODR", - Linkage::WeakAny => "WeakAny", - Linkage::WeakODR => "WeakODR", - Linkage::Appending => "Appending", - Linkage::Internal => "Internal", - Linkage::Private => "Private", - Linkage::ExternalWeak => "ExternalWeak", - Linkage::Common => "Common", - }; - - output.push('['); - output.push_str(linkage_abbrev); - output.push(']'); - } - output - }) - .collect(); - - item_keys.sort(); - - for item in item_keys { - println!("MONO_ITEM {item}"); - } - } - - (tcx.arena.alloc(mono_items), codegen_units) -} - -/// Outputs stats about instantiation counts and estimated size, per `MonoItem`'s -/// def, to a file in the given output directory. -fn dump_mono_items_stats<'tcx>( - tcx: TyCtxt<'tcx>, - codegen_units: &[CodegenUnit<'tcx>], - output_directory: &Option<PathBuf>, - crate_name: Symbol, -) -> Result<(), Box<dyn std::error::Error>> { - let output_directory = if let Some(ref directory) = output_directory { - fs::create_dir_all(directory)?; - directory - } else { - Path::new(".") - }; - - let format = tcx.sess.opts.unstable_opts.dump_mono_stats_format; - let ext = format.extension(); - let filename = format!("{crate_name}.mono_items.{ext}"); - let output_path = output_directory.join(&filename); - let file = File::create(&output_path)?; - let mut file = BufWriter::new(file); - - // Gather instantiated mono items grouped by def_id - let mut items_per_def_id: FxHashMap<_, Vec<_>> = Default::default(); - for cgu in codegen_units { - for (&mono_item, _) in cgu.items() { - // Avoid variable-sized compiler-generated shims - if mono_item.is_user_defined() { - items_per_def_id.entry(mono_item.def_id()).or_default().push(mono_item); - } - } - } - - #[derive(serde::Serialize)] - struct MonoItem { - name: String, - instantiation_count: usize, - size_estimate: usize, - total_estimate: usize, - } - - // Output stats sorted by total instantiated size, from heaviest to lightest - let mut stats: Vec<_> = items_per_def_id - .into_iter() - .map(|(def_id, items)| { - let name = with_no_trimmed_paths!(tcx.def_path_str(def_id)); - let instantiation_count = items.len(); - let size_estimate = items[0].size_estimate(tcx); - let total_estimate = instantiation_count * size_estimate; - MonoItem { name, instantiation_count, size_estimate, total_estimate } - }) - .collect(); - stats.sort_unstable_by_key(|item| cmp::Reverse(item.total_estimate)); - - if !stats.is_empty() { - match format { - DumpMonoStatsFormat::Json => serde_json::to_writer(file, &stats)?, - DumpMonoStatsFormat::Markdown => { - writeln!( - file, - "| Item | Instantiation count | Estimated Cost Per Instantiation | Total Estimated Cost |" - )?; - writeln!(file, "| --- | ---: | ---: | ---: |")?; - - for MonoItem { name, instantiation_count, size_estimate, total_estimate } in stats { - writeln!( - file, - "| `{name}` | {instantiation_count} | {size_estimate} | {total_estimate} |" - )?; - } - } - } - } - - Ok(()) -} - -fn codegened_and_inlined_items(tcx: TyCtxt<'_>, (): ()) -> &DefIdSet { - let (items, cgus) = tcx.collect_and_partition_mono_items(()); - let mut visited = DefIdSet::default(); - let mut result = items.clone(); - - for cgu in cgus { - for (item, _) in cgu.items() { - if let MonoItem::Fn(ref instance) = item { - let did = instance.def_id(); - if !visited.insert(did) { - continue; - } - let body = tcx.instance_mir(instance.def); - for block in body.basic_blocks.iter() { - for statement in &block.statements { - let mir::StatementKind::Coverage(_) = statement.kind else { continue }; - let scope = statement.source_info.scope; - if let Some(inlined) = scope.inlined_instance(&body.source_scopes) { - result.insert(inlined.def_id()); - } - } - } - } - } - } - - tcx.arena.alloc(result) -} - -pub fn provide(providers: &mut Providers) { - providers.collect_and_partition_mono_items = collect_and_partition_mono_items; - providers.codegened_and_inlined_items = codegened_and_inlined_items; - - providers.is_codegened_item = |tcx, def_id| { - let (all_mono_items, _) = tcx.collect_and_partition_mono_items(()); - all_mono_items.contains(&def_id) - }; - - providers.codegen_unit = |tcx, name| { - let (_, all) = tcx.collect_and_partition_mono_items(()); - all.iter() - .find(|cgu| cgu.name() == name) - .unwrap_or_else(|| panic!("failed to find cgu with name {name:?}")) - }; -} diff --git a/compiler/rustc_monomorphize/src/util.rs b/compiler/rustc_monomorphize/src/util.rs index 772f1520153..f6a80b0431f 100644 --- a/compiler/rustc_monomorphize/src/util.rs +++ b/compiler/rustc_monomorphize/src/util.rs @@ -29,12 +29,12 @@ pub(crate) fn dump_closure_profile<'tcx>(tcx: TyCtxt<'tcx>, closure_instance: In let before_feature_tys = tcx.subst_and_normalize_erasing_regions( closure_instance.substs, param_env, - ty::EarlyBinder::new(before_feature_tys), + ty::EarlyBinder::bind(before_feature_tys), ); let after_feature_tys = tcx.subst_and_normalize_erasing_regions( closure_instance.substs, param_env, - ty::EarlyBinder::new(after_feature_tys), + ty::EarlyBinder::bind(after_feature_tys), ); let new_size = tcx diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml index e596993465c..c9353b6fc9f 100644 --- a/compiler/rustc_query_impl/Cargo.toml +++ b/compiler/rustc_query_impl/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] -memoffset = { version = "0.6.0", features = ["unstable_const"] } +memoffset = { version = "0.8.0", features = ["unstable_const"] } field-offset = "0.3.5" measureme = "10.0.0" rustc_ast = { path = "../rustc_ast" } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 007e720823b..7cc2b2c880c 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1372,8 +1372,6 @@ options! { "set options for branch target identification and pointer authentication on AArch64"), cf_protection: CFProtection = (CFProtection::None, parse_cfprotection, [TRACKED], "instrument control-flow architecture protection"), - cgu_partitioning_strategy: Option<String> = (None, parse_opt_string, [TRACKED], - "the codegen unit partitioning strategy to use"), codegen_backend: Option<String> = (None, parse_opt_string, [TRACKED], "the backend to use"), combine_cgu: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 874d578fe1d..1185563ea80 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1454,6 +1454,10 @@ symbols! { stop_after_dataflow, store, str, + str_from_utf8, + str_from_utf8_mut, + str_from_utf8_unchecked, + str_from_utf8_unchecked_mut, str_split_whitespace, str_trim, str_trim_end, diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 55901bfbde9..0a805e2422d 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -274,7 +274,7 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> { let mut param_env = self.tcx.param_env_reveal_all_normalized(impl_def_id); if !substs.is_empty() { - param_env = EarlyBinder::new(param_env).subst(self.tcx, substs); + param_env = EarlyBinder::bind(param_env).subst(self.tcx, substs); } match &mut impl_trait_ref { diff --git a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs index 1071b8fc702..0f3f8f1ac2c 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs @@ -96,7 +96,7 @@ pub(in crate::solve) fn replace_erased_lifetimes_with_bound_vars<'tcx>( let br = ty::BoundRegion { var: ty::BoundVar::from_u32(counter), kind: ty::BrAnon(None) }; counter += 1; - tcx.mk_re_late_bound(current_depth, br) + ty::Region::new_late_bound(tcx, current_depth, br) } // All free regions should be erased here. r => bug!("unexpected region: {r:?}"), diff --git a/compiler/rustc_trait_selection/src/solve/canonicalize.rs b/compiler/rustc_trait_selection/src/solve/canonicalize.rs index ff4bff10cc8..29bdb5ff67d 100644 --- a/compiler/rustc_trait_selection/src/solve/canonicalize.rs +++ b/compiler/rustc_trait_selection/src/solve/canonicalize.rs @@ -255,7 +255,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> { }), ); let br = ty::BoundRegion { var, kind: BrAnon(None) }; - self.interner().mk_re_late_bound(self.binder_index, br) + ty::Region::new_late_bound(self.interner(), self.binder_index, br) } fn fold_ty(&mut self, mut t: Ty<'tcx>) -> Ty<'tcx> { diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs index fdb209fbff8..bca2343e424 100644 --- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs +++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs @@ -137,6 +137,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { #[instrument(level = "debug", skip(self), ret)] fn compute_external_query_constraints(&self) -> Result<ExternalConstraints<'tcx>, NoSolution> { + // We only check for leaks from universes which were entered inside + // of the query. + self.infcx.leak_check(self.max_input_universe, None).map_err(|e| { + debug!(?e, "failed the leak check"); + NoSolution + })?; + // Cannot use `take_registered_region_obligations` as we may compute the response // inside of a `probe` whenever we have multiple choices inside of the solver. let region_obligations = self.infcx.inner.borrow().region_obligations().to_owned(); diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs index e8c5a8fab2a..d6fd457de06 100644 --- a/compiler/rustc_trait_selection/src/traits/coherence.rs +++ b/compiler/rustc_trait_selection/src/traits/coherence.rs @@ -5,7 +5,7 @@ //! [trait-specialization]: https://rustc-dev-guide.rust-lang.org/traits/specialization.html use crate::infer::outlives::env::OutlivesEnvironment; -use crate::infer::{CombinedSnapshot, InferOk}; +use crate::infer::InferOk; use crate::traits::outlives_bounds::InferCtxtExt as _; use crate::traits::select::IntercrateAmbiguityCause; use crate::traits::util::impl_subject_and_oblig; @@ -62,6 +62,21 @@ pub fn add_placeholder_note(err: &mut Diagnostic) { ); } +#[derive(Debug, Clone, Copy)] +enum TrackAmbiguityCauses { + Yes, + No, +} + +impl TrackAmbiguityCauses { + fn is_yes(self) -> bool { + match self { + TrackAmbiguityCauses::Yes => true, + TrackAmbiguityCauses::No => false, + } + } +} + /// If there are types that satisfy both impls, returns `Some` /// with a suitably-freshened `ImplHeader` with those types /// substituted. Otherwise, returns `None`. @@ -97,29 +112,28 @@ pub fn overlapping_impls( return None; } - let infcx = tcx - .infer_ctxt() - .with_opaque_type_inference(DefiningAnchor::Bubble) - .intercrate(true) - .build(); - let selcx = &mut SelectionContext::new(&infcx); - let overlaps = - overlap(selcx, skip_leak_check, impl1_def_id, impl2_def_id, overlap_mode).is_some(); - if !overlaps { - return None; - } + let _overlap_with_bad_diagnostics = overlap( + tcx, + TrackAmbiguityCauses::No, + skip_leak_check, + impl1_def_id, + impl2_def_id, + overlap_mode, + )?; // In the case where we detect an error, run the check again, but // this time tracking intercrate ambiguity causes for better // diagnostics. (These take time and can lead to false errors.) - let infcx = tcx - .infer_ctxt() - .with_opaque_type_inference(DefiningAnchor::Bubble) - .intercrate(true) - .build(); - let selcx = &mut SelectionContext::new(&infcx); - selcx.enable_tracking_intercrate_ambiguity_causes(); - Some(overlap(selcx, skip_leak_check, impl1_def_id, impl2_def_id, overlap_mode).unwrap()) + let overlap = overlap( + tcx, + TrackAmbiguityCauses::Yes, + skip_leak_check, + impl1_def_id, + impl2_def_id, + overlap_mode, + ) + .unwrap(); + Some(overlap) } fn with_fresh_ty_vars<'cx, 'tcx>( @@ -146,40 +160,34 @@ fn with_fresh_ty_vars<'cx, 'tcx>( /// Can both impl `a` and impl `b` be satisfied by a common type (including /// where-clauses)? If so, returns an `ImplHeader` that unifies the two impls. -fn overlap<'cx, 'tcx>( - selcx: &mut SelectionContext<'cx, 'tcx>, +#[instrument(level = "debug", skip(tcx))] +fn overlap<'tcx>( + tcx: TyCtxt<'tcx>, + track_ambiguity_causes: TrackAmbiguityCauses, skip_leak_check: SkipLeakCheck, impl1_def_id: DefId, impl2_def_id: DefId, overlap_mode: OverlapMode, ) -> Option<OverlapResult<'tcx>> { - debug!( - "overlap(impl1_def_id={:?}, impl2_def_id={:?}, overlap_mode={:?})", - impl1_def_id, impl2_def_id, overlap_mode - ); - - selcx.infcx.probe_maybe_skip_leak_check(skip_leak_check.is_yes(), |snapshot| { - overlap_within_probe(selcx, impl1_def_id, impl2_def_id, overlap_mode, snapshot) - }) -} - -fn overlap_within_probe<'cx, 'tcx>( - selcx: &mut SelectionContext<'cx, 'tcx>, - impl1_def_id: DefId, - impl2_def_id: DefId, - overlap_mode: OverlapMode, - snapshot: &CombinedSnapshot<'tcx>, -) -> Option<OverlapResult<'tcx>> { - let infcx = selcx.infcx; - if overlap_mode.use_negative_impl() { - if negative_impl(infcx.tcx, impl1_def_id, impl2_def_id) - || negative_impl(infcx.tcx, impl2_def_id, impl1_def_id) + if negative_impl(tcx, impl1_def_id, impl2_def_id) + || negative_impl(tcx, impl2_def_id, impl1_def_id) { return None; } } + let infcx = tcx + .infer_ctxt() + .with_opaque_type_inference(DefiningAnchor::Bubble) + .skip_leak_check(skip_leak_check.is_yes()) + .intercrate(true) + .build(); + let selcx = &mut SelectionContext::new(&infcx); + if track_ambiguity_causes.is_yes() { + selcx.enable_tracking_intercrate_ambiguity_causes(); + } + // For the purposes of this check, we don't bring any placeholder // types into scope; instead, we replace the generic types with // fresh type variables, and hence we do our evaluations in an @@ -198,18 +206,23 @@ fn overlap_within_probe<'cx, 'tcx>( } } - // We disable the leak when creating the `snapshot` by using - // `infcx.probe_maybe_disable_leak_check`. - if infcx.leak_check(true, snapshot).is_err() { + // We toggle the `leak_check` by using `skip_leak_check` when constructing the + // inference context, so this may be a noop. + if infcx.leak_check(ty::UniverseIndex::ROOT, None).is_err() { debug!("overlap: leak check failed"); return None; } let intercrate_ambiguity_causes = selcx.take_intercrate_ambiguity_causes(); debug!("overlap: intercrate_ambiguity_causes={:#?}", intercrate_ambiguity_causes); - - let involves_placeholder = - matches!(selcx.infcx.region_constraints_added_in_snapshot(snapshot), Some(true)); + let involves_placeholder = infcx + .inner + .borrow_mut() + .unwrap_region_constraints() + .data() + .constraints + .iter() + .any(|c| c.0.involves_placeholders()); let impl_header = selcx.infcx.resolve_vars_if_possible(impl1_header); Some(OverlapResult { impl_header, intercrate_ambiguity_causes, involves_placeholder }) diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs index 0f84032d9ef..f7389bda159 100644 --- a/compiler/rustc_trait_selection/src/traits/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/mod.rs @@ -32,7 +32,7 @@ use rustc_errors::ErrorGuaranteed; use rustc_middle::query::Providers; use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt}; -use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt, TypeSuperVisitable}; +use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt, TypeFolder, TypeSuperVisitable}; use rustc_middle::ty::{InternalSubsts, SubstsRef}; use rustc_span::def_id::DefId; use rustc_span::Span; @@ -272,8 +272,62 @@ pub fn normalize_param_env_or_error<'tcx>( // parameter environments once for every fn as it goes, // and errors will get reported then; so outside of type inference we // can be sure that no errors should occur. - let mut predicates: Vec<_> = - util::elaborate(tcx, unnormalized_env.caller_bounds().into_iter()).collect(); + let mut predicates: Vec<_> = util::elaborate( + tcx, + unnormalized_env.caller_bounds().into_iter().map(|predicate| { + if tcx.features().generic_const_exprs { + return predicate; + } + + struct ConstNormalizer<'tcx>(TyCtxt<'tcx>); + + impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ConstNormalizer<'tcx> { + fn interner(&self) -> TyCtxt<'tcx> { + self.0 + } + + fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> { + // While it is pretty sus to be evaluating things with an empty param env, it + // should actually be okay since without `feature(generic_const_exprs)` the only + // const arguments that have a non-empty param env are array repeat counts. These + // do not appear in the type system though. + c.eval(self.0, ty::ParamEnv::empty()) + } + } + + // This whole normalization step is a hack to work around the fact that + // `normalize_param_env_or_error` is fundamentally broken from using an + // unnormalized param env with a trait solver that expects the param env + // to be normalized. + // + // When normalizing the param env we can end up evaluating obligations + // that have been normalized but can only be proven via a where clause + // which is still in its unnormalized form. example: + // + // Attempting to prove `T: Trait<<u8 as Identity>::Assoc>` in a param env + // with a `T: Trait<<u8 as Identity>::Assoc>` where clause will fail because + // we first normalize obligations before proving them so we end up proving + // `T: Trait<u8>`. Since lazy normalization is not implemented equating `u8` + // with `<u8 as Identity>::Assoc` fails outright so we incorrectly believe that + // we cannot prove `T: Trait<u8>`. + // + // The same thing is true for const generics- attempting to prove + // `T: Trait<ConstKind::Unevaluated(...)>` with the same thing as a where clauses + // will fail. After normalization we may be attempting to prove `T: Trait<4>` with + // the unnormalized where clause `T: Trait<ConstKind::Unevaluated(...)>`. In order + // for the obligation to hold `4` must be equal to `ConstKind::Unevaluated(...)` + // but as we do not have lazy norm implemented, equating the two consts fails outright. + // + // Ideally we would not normalize consts here at all but it is required for backwards + // compatibility. Eventually when lazy norm is implemented this can just be removed. + // We do not normalize types here as there is no backwards compatibility requirement + // for us to do so. + // + // FIXME(-Ztrait-solver=next): remove this hack since we have deferred projection equality + predicate.fold_with(&mut ConstNormalizer(tcx)) + }), + ) + .collect(); debug!("normalize_param_env_or_error: elaborated-predicates={:?}", predicates); @@ -487,7 +541,7 @@ fn is_impossible_method(tcx: TyCtxt<'_>, (impl_def_id, trait_item_def_id): (DefI tcx, ObligationCause::dummy_with_span(*span), param_env, - ty::EarlyBinder::new(*pred).subst(tcx, impl_trait_ref.substs), + ty::EarlyBinder::bind(*pred).subst(tcx, impl_trait_ref.substs), ) }) }); diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs index 0be21156b6c..9582479941b 100644 --- a/compiler/rustc_trait_selection/src/traits/object_safety.rs +++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs @@ -642,7 +642,7 @@ fn receiver_for_self_ty<'tcx>( if param.index == 0 { self_ty.into() } else { tcx.mk_param_from_def(param) } }); - let result = EarlyBinder::new(receiver_ty).subst(tcx, substs); + let result = EarlyBinder::bind(receiver_ty).subst(tcx, substs); debug!( "receiver_for_self_ty({:?}, {:?}, {:?}) = {:?}", receiver_ty, self_ty, method_def_id, result diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index 51069897120..65af0bb1c4e 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -824,7 +824,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> { let universe = self.universe_for(debruijn); let p = ty::PlaceholderRegion { universe, bound: br }; self.mapped_regions.insert(p, br); - self.infcx.tcx.mk_re_placeholder(p) + ty::Region::new_placeholder(self.infcx.tcx, p) } _ => r, } @@ -945,7 +945,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> { let db = ty::DebruijnIndex::from_usize( self.universe_indices.len() - index + self.current_index.as_usize() - 1, ); - self.interner().mk_re_late_bound(db, *replace_var) + ty::Region::new_late_bound(self.interner(), db, *replace_var) } None => r1, } diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index 0a1b1b10b00..709c3f432e6 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -307,13 +307,13 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( // there, but that needs some way to handle cycles. constraints .dtorck_types - .extend(dtorck_types.iter().map(|t| EarlyBinder::new(*t).subst(tcx, substs))); + .extend(dtorck_types.iter().map(|t| EarlyBinder::bind(*t).subst(tcx, substs))); constraints .outlives - .extend(outlives.iter().map(|t| EarlyBinder::new(*t).subst(tcx, substs))); + .extend(outlives.iter().map(|t| EarlyBinder::bind(*t).subst(tcx, substs))); constraints .overflows - .extend(overflows.iter().map(|t| EarlyBinder::new(*t).subst(tcx, substs))); + .extend(overflows.iter().map(|t| EarlyBinder::bind(*t).subst(tcx, substs))); } // Objects must be alive in order for their destructor diff --git a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs index edbe2de8105..a8a74d7501a 100644 --- a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs +++ b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs @@ -90,7 +90,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> { Ok(EvaluationResult::EvaluatedToAmbig) } else if self.opaque_types_added_in_snapshot(snapshot) { Ok(EvaluationResult::EvaluatedToOkModuloOpaqueTypes) - } else if self.region_constraints_added_in_snapshot(snapshot).is_some() { + } else if self.region_constraints_added_in_snapshot(snapshot) { Ok(EvaluationResult::EvaluatedToOkModuloRegions) } else { Ok(EvaluationResult::EvaluatedToOk) diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index 0245dfd333b..4e961c3ee73 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -550,7 +550,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name); let bound_var = ty::BoundVariableKind::Region(kind); bound_vars.push(bound_var); - tcx.mk_re_late_bound( + ty::Region::new_late_bound( + tcx, ty::INNERMOST, ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index ed380f32b72..42c1b629ac2 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -561,9 +561,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { op: impl FnOnce(&mut Self) -> Result<EvaluationResult, OverflowError>, ) -> Result<EvaluationResult, OverflowError> { self.infcx.probe(|snapshot| -> Result<EvaluationResult, OverflowError> { + let outer_universe = self.infcx.universe(); let result = op(self)?; - match self.infcx.leak_check(true, snapshot) { + match self.infcx.leak_check(outer_universe, Some(snapshot)) { Ok(()) => {} Err(_) => return Ok(EvaluatedToErr), } @@ -572,9 +573,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { return Ok(result.max(EvaluatedToOkModuloOpaqueTypes)); } - match self.infcx.region_constraints_added_in_snapshot(snapshot) { - None => Ok(result), - Some(_) => Ok(result.max(EvaluatedToOkModuloRegions)), + if self.infcx.region_constraints_added_in_snapshot(snapshot) { + Ok(result.max(EvaluatedToOkModuloRegions)) + } else { + Ok(result) } }) } @@ -3027,7 +3029,7 @@ fn bind_generator_hidden_types_above<'tcx>( kind: ty::BrAnon(None), }; counter += 1; - tcx.mk_re_late_bound(current_depth, br) + ty::Region::new_late_bound(tcx, current_depth, br) } r => bug!("unexpected region: {r:?}"), }) diff --git a/compiler/rustc_traits/src/chalk/db.rs b/compiler/rustc_traits/src/chalk/db.rs index 1d3ea96666f..38f94c38861 100644 --- a/compiler/rustc_traits/src/chalk/db.rs +++ b/compiler/rustc_traits/src/chalk/db.rs @@ -727,7 +727,7 @@ fn bound_vars_for_item(tcx: TyCtxt<'_>, def_id: DefId) -> SubstsRef<'_> { var: ty::BoundVar::from_usize(substs.len()), kind: ty::BrAnon(None), }; - tcx.mk_re_late_bound(ty::INNERMOST, br).into() + ty::Region::new_late_bound(tcx, ty::INNERMOST, br).into() } ty::GenericParamDefKind::Const { .. } => tcx diff --git a/compiler/rustc_traits/src/chalk/lowering.rs b/compiler/rustc_traits/src/chalk/lowering.rs index e447ab94f64..e6c6e0f13df 100644 --- a/compiler/rustc_traits/src/chalk/lowering.rs +++ b/compiler/rustc_traits/src/chalk/lowering.rs @@ -542,7 +542,8 @@ impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'t fn lower_into(self, interner: RustInterner<'tcx>) -> Region<'tcx> { let tcx = interner.tcx; match self.data(interner) { - chalk_ir::LifetimeData::BoundVar(var) => tcx.mk_re_late_bound( + chalk_ir::LifetimeData::BoundVar(var) => ty::Region::new_late_bound( + tcx, ty::DebruijnIndex::from_u32(var.debruijn.depth()), ty::BoundRegion { var: ty::BoundVar::from_usize(var.index), @@ -550,13 +551,16 @@ impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'t }, ), chalk_ir::LifetimeData::InferenceVar(_var) => unimplemented!(), - chalk_ir::LifetimeData::Placeholder(p) => tcx.mk_re_placeholder(ty::Placeholder { - universe: ty::UniverseIndex::from_usize(p.ui.counter), - bound: ty::BoundRegion { - var: ty::BoundVar::from_usize(p.idx), - kind: ty::BoundRegionKind::BrAnon(None), + chalk_ir::LifetimeData::Placeholder(p) => ty::Region::new_placeholder( + tcx, + ty::Placeholder { + universe: ty::UniverseIndex::from_usize(p.ui.counter), + bound: ty::BoundRegion { + var: ty::BoundVar::from_usize(p.idx), + kind: ty::BoundRegionKind::BrAnon(None), + }, }, - }), + ), chalk_ir::LifetimeData::Static => tcx.lifetimes.re_static, chalk_ir::LifetimeData::Erased => tcx.lifetimes.re_erased, chalk_ir::LifetimeData::Phantom(void, _) => match *void {}, @@ -1051,7 +1055,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for NamedBoundVarSubstitutor<'a, 'tcx> { ty::BrNamed(def_id, _name) => match self.named_parameters.get(&def_id) { Some(_) => { let new_br = ty::BoundRegion { var: br.var, kind: ty::BrAnon(None) }; - return self.tcx.mk_re_late_bound(index, new_br); + return ty::Region::new_late_bound(self.tcx, index, new_br); } None => panic!("Missing `BrNamed`."), }, @@ -1142,7 +1146,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> { var: ty::BoundVar::from_u32(*idx), kind: ty::BrAnon(None), }; - self.tcx.mk_re_late_bound(self.binder_index, br) + ty::Region::new_late_bound(self.tcx, self.binder_index, br) } None => { let idx = self.named_regions.len() as u32; @@ -1151,7 +1155,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> { kind: ty::BrAnon(None), }; self.named_regions.insert(_re.def_id, idx); - self.tcx.mk_re_late_bound(self.binder_index, br) + ty::Region::new_late_bound(self.tcx, self.binder_index, br) } }, diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 15c19104616..5b3ffc9fc36 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -81,7 +81,7 @@ fn fn_sig_for_fn_abi<'tcx>( var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BoundRegionKind::BrEnv, }; - let env_region = tcx.mk_re_late_bound(ty::INNERMOST, br); + let env_region = ty::Region::new_late_bound(tcx, ty::INNERMOST, br); let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap(); let sig = sig.skip_binder(); @@ -106,7 +106,7 @@ fn fn_sig_for_fn_abi<'tcx>( var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind: ty::BoundRegionKind::BrEnv, }; - let env_ty = tcx.mk_mut_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), ty); + let env_ty = tcx.mk_mut_ref(ty::Region::new_late_bound(tcx, ty::INNERMOST, br), ty); let pin_did = tcx.require_lang_item(LangItem::Pin, None); let pin_adt_ref = tcx.adt_def(pin_did); diff --git a/compiler/rustc_ty_utils/src/assoc.rs b/compiler/rustc_ty_utils/src/assoc.rs index 0925b3956de..57b4183d336 100644 --- a/compiler/rustc_ty_utils/src/assoc.rs +++ b/compiler/rustc_ty_utils/src/assoc.rs @@ -301,7 +301,7 @@ fn associated_type_for_impl_trait_in_trait( trait_assoc_ty.impl_defaultness(tcx.impl_defaultness(fn_def_id)); // Copy type_of of the opaque. - trait_assoc_ty.type_of(ty::EarlyBinder::new(tcx.mk_opaque( + trait_assoc_ty.type_of(ty::EarlyBinder::bind(tcx.mk_opaque( opaque_ty_def_id.to_def_id(), InternalSubsts::identity_for_item(tcx, opaque_ty_def_id), ))); diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs index bb723c9997a..ce77df0df5d 100644 --- a/compiler/rustc_ty_utils/src/consts.rs +++ b/compiler/rustc_ty_utils/src/consts.rs @@ -419,7 +419,7 @@ pub fn thir_abstract_const( let root_span = body.exprs[body_id].span; - Ok(Some(ty::EarlyBinder::new(recurse_build(tcx, body, body_id, root_span)?))) + Ok(Some(ty::EarlyBinder::bind(recurse_build(tcx, body, body_id, root_span)?))) } pub fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 0e5b23ca423..7015778e24b 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -610,7 +610,7 @@ fn generator_layout<'tcx>( ) -> Result<Layout<'tcx>, LayoutError<'tcx>> { use SavedLocalEligibility::*; let tcx = cx.tcx; - let subst_field = |ty: Ty<'tcx>| EarlyBinder::new(ty).subst(tcx, substs); + let subst_field = |ty: Ty<'tcx>| EarlyBinder::bind(ty).subst(tcx, substs); let Some(info) = tcx.generator_layout(def_id) else { return Err(LayoutError::Unknown(ty)); diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 075fde6ddb6..9d593dc5e04 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -210,7 +210,7 @@ fn drop_tys_helper<'tcx>( match subty.kind() { ty::Adt(adt_id, subst) => { for subty in tcx.adt_drop_tys(adt_id.did())? { - vec.push(EarlyBinder::new(subty).subst(tcx, subst)); + vec.push(EarlyBinder::bind(subty).subst(tcx, subst)); } } _ => vec.push(subty), diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs index 52bc386cb00..12ad8b0842f 100644 --- a/compiler/rustc_ty_utils/src/ty.rs +++ b/compiler/rustc_ty_utils/src/ty.rs @@ -287,12 +287,13 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> { let shifted_alias_ty = self.tcx.fold_regions(unshifted_alias_ty, |re, depth| { if let ty::ReLateBound(index, bv) = re.kind() { if depth != ty::INNERMOST { - return self.tcx.mk_re_error_with_message( + return ty::Region::new_error_with_message( + self.tcx, DUMMY_SP, "we shouldn't walk non-predicate binders with `impl Trait`...", ); } - self.tcx.mk_re_late_bound(index.shifted_out_to_binder(self.depth), bv) + ty::Region::new_late_bound(self.tcx, index.shifted_out_to_binder(self.depth), bv) } else { re } @@ -506,7 +507,7 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<EarlyBinder<Ty<' if self_ty_matches { debug!("issue33140_self_ty - MATCHES!"); - Some(EarlyBinder::new(self_ty)) + Some(EarlyBinder::bind(self_ty)) } else { debug!("issue33140_self_ty - non-matching self type"); None diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs index f6b44bdf27e..ae16fbb162e 100644 --- a/compiler/rustc_type_ir/src/lib.rs +++ b/compiler/rustc_type_ir/src/lib.rs @@ -52,7 +52,7 @@ pub trait Interner: Sized { type PolyFnSig: Clone + Debug + Hash + Ord; type ListBinderExistentialPredicate: Clone + Debug + Hash + Ord; type BinderListTy: Clone + Debug + Hash + Ord; - type ListTy: Clone + Debug + Hash + Ord; + type ListTy: Clone + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>; type AliasTy: Clone + Debug + Hash + Ord; type ParamTy: Clone + Debug + Hash + Ord; type BoundTy: Clone + Debug + Hash + Ord; @@ -67,6 +67,9 @@ pub trait Interner: Sized { type FreeRegion: Clone + Debug + Hash + Ord; type RegionVid: Clone + Debug + Hash + Ord; type PlaceholderRegion: Clone + Debug + Hash + Ord; + + fn ty_and_mut_to_parts(ty_and_mut: Self::TypeAndMut) -> (Self::Ty, Self::Mutability); + fn mutability_is_mut(mutbl: Self::Mutability) -> bool; } /// Imagine you have a function `F: FnOnce(&[T]) -> R`, plus an iterator `iter` @@ -390,7 +393,19 @@ impl DebruijnIndex { } } -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub fn debug_bound_var<T: std::fmt::Write>( + fmt: &mut T, + debruijn: DebruijnIndex, + var: impl std::fmt::Debug, +) -> Result<(), std::fmt::Error> { + if debruijn == INNERMOST { + write!(fmt, "^{:?}", var) + } else { + write!(fmt, "^{}_{:?}", debruijn.index(), var) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Encodable, Decodable, HashStable_Generic)] pub enum IntTy { Isize, @@ -448,7 +463,7 @@ impl IntTy { } } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy, Debug)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy)] #[derive(Encodable, Decodable, HashStable_Generic)] pub enum UintTy { Usize, @@ -506,7 +521,7 @@ impl UintTy { } } -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[derive(Encodable, Decodable, HashStable_Generic)] pub enum FloatTy { F32, diff --git a/compiler/rustc_type_ir/src/structural_impls.rs b/compiler/rustc_type_ir/src/structural_impls.rs index 45a2e9023c9..553d7f31b2d 100644 --- a/compiler/rustc_type_ir/src/structural_impls.rs +++ b/compiler/rustc_type_ir/src/structural_impls.rs @@ -4,11 +4,12 @@ use crate::fold::{FallibleTypeFolder, TypeFoldable}; use crate::visit::{TypeVisitable, TypeVisitor}; -use crate::Interner; +use crate::{FloatTy, IntTy, Interner, UintTy}; use rustc_data_structures::functor::IdFunctor; use rustc_data_structures::sync::Lrc; use rustc_index::{Idx, IndexVec}; +use core::fmt; use std::ops::ControlFlow; /////////////////////////////////////////////////////////////////////////// @@ -163,3 +164,21 @@ impl<I: Interner, T: TypeVisitable<I>, Ix: Idx> TypeVisitable<I> for IndexVec<Ix self.iter().try_for_each(|t| t.visit_with(visitor)) } } + +impl fmt::Debug for IntTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} + +impl fmt::Debug for UintTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} + +impl fmt::Debug for FloatTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} diff --git a/compiler/rustc_type_ir/src/sty.rs b/compiler/rustc_type_ir/src/sty.rs index f7344bacc02..fa18f921ee4 100644 --- a/compiler/rustc_type_ir/src/sty.rs +++ b/compiler/rustc_type_ir/src/sty.rs @@ -294,7 +294,7 @@ impl<I: Interner> Clone for TyKind<I> { Str => Str, Array(t, c) => Array(t.clone(), c.clone()), Slice(t) => Slice(t.clone()), - RawPtr(t) => RawPtr(t.clone()), + RawPtr(p) => RawPtr(p.clone()), Ref(r, t, m) => Ref(r.clone(), t.clone(), m.clone()), FnDef(d, s) => FnDef(d.clone(), s.clone()), FnPtr(s) => FnPtr(s.clone()), @@ -499,33 +499,65 @@ impl<I: Interner> hash::Hash for TyKind<I> { impl<I: Interner> fmt::Debug for TyKind<I> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Bool => f.write_str("Bool"), - Char => f.write_str("Char"), - Int(i) => f.debug_tuple_field1_finish("Int", i), - Uint(u) => f.debug_tuple_field1_finish("Uint", u), - Float(float) => f.debug_tuple_field1_finish("Float", float), + Bool => write!(f, "bool"), + Char => write!(f, "char"), + Int(i) => write!(f, "{i:?}"), + Uint(u) => write!(f, "{u:?}"), + Float(float) => write!(f, "{float:?}"), Adt(d, s) => f.debug_tuple_field2_finish("Adt", d, s), Foreign(d) => f.debug_tuple_field1_finish("Foreign", d), - Str => f.write_str("Str"), - Array(t, c) => f.debug_tuple_field2_finish("Array", t, c), - Slice(t) => f.debug_tuple_field1_finish("Slice", t), - RawPtr(t) => f.debug_tuple_field1_finish("RawPtr", t), - Ref(r, t, m) => f.debug_tuple_field3_finish("Ref", r, t, m), + Str => write!(f, "str"), + Array(t, c) => write!(f, "[{t:?}; {c:?}]"), + Slice(t) => write!(f, "[{t:?}]"), + RawPtr(p) => { + let (ty, mutbl) = I::ty_and_mut_to_parts(p.clone()); + match I::mutability_is_mut(mutbl) { + true => write!(f, "*mut "), + false => write!(f, "*const "), + }?; + write!(f, "{ty:?}") + } + Ref(r, t, m) => match I::mutability_is_mut(m.clone()) { + true => write!(f, "&{r:?} mut {t:?}"), + false => write!(f, "&{r:?} {t:?}"), + }, FnDef(d, s) => f.debug_tuple_field2_finish("FnDef", d, s), - FnPtr(s) => f.debug_tuple_field1_finish("FnPtr", s), - Dynamic(p, r, repr) => f.debug_tuple_field3_finish("Dynamic", p, r, repr), + FnPtr(s) => write!(f, "{s:?}"), + Dynamic(p, r, repr) => match repr { + DynKind::Dyn => write!(f, "dyn {p:?} + {r:?}"), + DynKind::DynStar => write!(f, "dyn* {p:?} + {r:?}"), + }, Closure(d, s) => f.debug_tuple_field2_finish("Closure", d, s), Generator(d, s, m) => f.debug_tuple_field3_finish("Generator", d, s, m), GeneratorWitness(g) => f.debug_tuple_field1_finish("GeneratorWitness", g), GeneratorWitnessMIR(d, s) => f.debug_tuple_field2_finish("GeneratorWitnessMIR", d, s), - Never => f.write_str("Never"), - Tuple(t) => f.debug_tuple_field1_finish("Tuple", t), + Never => write!(f, "!"), + Tuple(t) => { + let mut iter = t.clone().into_iter(); + + write!(f, "(")?; + + match iter.next() { + None => return write!(f, ")"), + Some(ty) => write!(f, "{ty:?}")?, + }; + + match iter.next() { + None => return write!(f, ",)"), + Some(ty) => write!(f, "{ty:?})")?, + } + + for ty in iter { + write!(f, ", {ty:?}")?; + } + write!(f, ")") + } Alias(i, a) => f.debug_tuple_field2_finish("Alias", i, a), - Param(p) => f.debug_tuple_field1_finish("Param", p), - Bound(d, b) => f.debug_tuple_field2_finish("Bound", d, b), - Placeholder(p) => f.debug_tuple_field1_finish("Placeholder", p), - Infer(t) => f.debug_tuple_field1_finish("Infer", t), - TyKind::Error(e) => f.debug_tuple_field1_finish("Error", e), + Param(p) => write!(f, "{p:?}"), + Bound(d, b) => crate::debug_bound_var(f, *d, b), + Placeholder(p) => write!(f, "{p:?}"), + Infer(t) => write!(f, "{t:?}"), + TyKind::Error(_) => write!(f, "{{type error}}"), } } } |