diff options
Diffstat (limited to 'compiler/rustc_middle/src/ty')
| -rw-r--r-- | compiler/rustc_middle/src/ty/consts.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/context.rs | 66 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/fold.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/layout.rs | 34 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/mod.rs | 225 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/print/pretty.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/region.rs | 36 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/structural_impls.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/sty.rs | 22 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/typeck_results.rs | 16 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/util.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/visit.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_middle/src/ty/vtable.rs | 2 |
13 files changed, 412 insertions, 42 deletions
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 614b6471f18..95adb561c70 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -95,7 +95,15 @@ impl<'tcx> Const<'tcx> { debruijn: ty::DebruijnIndex, bound_const: ty::BoundConst, ) -> Const<'tcx> { - Const::new(tcx, ty::ConstKind::Bound(debruijn, bound_const)) + Const::new(tcx, ty::ConstKind::Bound(ty::BoundVarIndexKind::Bound(debruijn), bound_const)) + } + + #[inline] + pub fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: ty::BoundVar) -> Const<'tcx> { + Const::new( + tcx, + ty::ConstKind::Bound(ty::BoundVarIndexKind::Canonical, ty::BoundConst { var }), + ) } #[inline] @@ -180,6 +188,10 @@ impl<'tcx> rustc_type_ir::inherent::Const<TyCtxt<'tcx>> for Const<'tcx> { Const::new_bound(tcx, debruijn, ty::BoundConst { var }) } + fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: rustc_type_ir::BoundVar) -> Self { + Const::new_canonical_bound(tcx, var) + } + fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderConst) -> Self { Const::new_placeholder(tcx, placeholder) } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 7d3e2c9965d..1b89a49cf98 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -75,7 +75,7 @@ use crate::thir::Thir; use crate::traits; use crate::traits::solve::{ self, CanonicalInput, ExternalConstraints, ExternalConstraintsData, PredefinedOpaques, - PredefinedOpaquesData, QueryResult, inspect, + QueryResult, inspect, }; use crate::ty::predicate::ExistentialPredicateStableCmpExt as _; use crate::ty::{ @@ -116,7 +116,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { fn mk_predefined_opaques_in_body( self, - data: PredefinedOpaquesData<Self>, + data: &[(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)], ) -> Self::PredefinedOpaques { self.mk_predefined_opaques_in_body(data) } @@ -941,7 +941,7 @@ pub struct CtxtInterners<'tcx> { layout: InternedSet<'tcx, LayoutData<FieldIdx, VariantIdx>>, adt_def: InternedSet<'tcx, AdtDefData>, external_constraints: InternedSet<'tcx, ExternalConstraintsData<TyCtxt<'tcx>>>, - predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<TyCtxt<'tcx>>>, + predefined_opaques_in_body: InternedSet<'tcx, List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>>, fields: InternedSet<'tcx, List<FieldIdx>>, local_def_ids: InternedSet<'tcx, List<LocalDefId>>, captures: InternedSet<'tcx, List<&'tcx ty::CapturedPlace<'tcx>>>, @@ -1110,6 +1110,15 @@ const NUM_PREINTERNED_FRESH_TYS: u32 = 20; const NUM_PREINTERNED_FRESH_INT_TYS: u32 = 3; const NUM_PREINTERNED_FRESH_FLOAT_TYS: u32 = 3; const NUM_PREINTERNED_ANON_BOUND_TYS_I: u32 = 3; + +// From general profiling of the *max vars during canonicalization* of a value: +// - about 90% of the time, there are no canonical vars +// - about 9% of the time, there is only one canonical var +// - there are rarely more than 3-5 canonical vars (with exceptions in particularly pathological cases) +// This may not match the number of bound vars found in `for`s. +// Given that this is all heap interned, it seems likely that interning fewer +// vars here won't make an appreciable difference. Though, if we were to inline the data (in an array), +// we may want to consider reducing the number for canonicalized vars down to 4 or so. const NUM_PREINTERNED_ANON_BOUND_TYS_V: u32 = 20; // This number may seem high, but it is reached in all but the smallest crates. @@ -1160,9 +1169,14 @@ pub struct CommonTypes<'tcx> { pub fresh_float_tys: Vec<Ty<'tcx>>, /// Pre-interned values of the form: - /// `Bound(DebruijnIndex(i), BoundTy { var: v, kind: BoundTyKind::Anon})` + /// `Bound(BoundVarIndexKind::Bound(DebruijnIndex(i)), BoundTy { var: v, kind: BoundTyKind::Anon})` /// for small values of `i` and `v`. pub anon_bound_tys: Vec<Vec<Ty<'tcx>>>, + + // Pre-interned values of the form: + // `Bound(BoundVarIndexKind::Canonical, BoundTy { var: v, kind: BoundTyKind::Anon })` + // for small values of `v`. + pub anon_canonical_bound_tys: Vec<Ty<'tcx>>, } pub struct CommonLifetimes<'tcx> { @@ -1176,9 +1190,14 @@ pub struct CommonLifetimes<'tcx> { pub re_vars: Vec<Region<'tcx>>, /// Pre-interned values of the form: - /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BoundRegionKind::Anon })` + /// `ReBound(BoundVarIndexKind::Bound(DebruijnIndex(i)), BoundRegion { var: v, kind: BoundRegionKind::Anon })` /// for small values of `i` and `v`. pub anon_re_bounds: Vec<Vec<Region<'tcx>>>, + + // Pre-interned values of the form: + // `ReBound(BoundVarIndexKind::Canonical, BoundRegion { var: v, kind: BoundRegionKind::Anon })` + // for small values of `v`. + pub anon_re_canonical_bounds: Vec<Region<'tcx>>, } pub struct CommonConsts<'tcx> { @@ -1211,7 +1230,7 @@ impl<'tcx> CommonTypes<'tcx> { (0..NUM_PREINTERNED_ANON_BOUND_TYS_V) .map(|v| { mk(ty::Bound( - ty::DebruijnIndex::from(i), + ty::BoundVarIndexKind::Bound(ty::DebruijnIndex::from(i)), ty::BoundTy { var: ty::BoundVar::from(v), kind: ty::BoundTyKind::Anon }, )) }) @@ -1219,6 +1238,15 @@ impl<'tcx> CommonTypes<'tcx> { }) .collect(); + let anon_canonical_bound_tys = (0..NUM_PREINTERNED_ANON_BOUND_TYS_V) + .map(|v| { + mk(ty::Bound( + ty::BoundVarIndexKind::Canonical, + ty::BoundTy { var: ty::BoundVar::from(v), kind: ty::BoundTyKind::Anon }, + )) + }) + .collect(); + CommonTypes { unit: mk(Tuple(List::empty())), bool: mk(Bool), @@ -1250,6 +1278,7 @@ impl<'tcx> CommonTypes<'tcx> { fresh_int_tys, fresh_float_tys, anon_bound_tys, + anon_canonical_bound_tys, } } } @@ -1270,7 +1299,7 @@ impl<'tcx> CommonLifetimes<'tcx> { (0..NUM_PREINTERNED_ANON_RE_BOUNDS_V) .map(|v| { mk(ty::ReBound( - ty::DebruijnIndex::from(i), + ty::BoundVarIndexKind::Bound(ty::DebruijnIndex::from(i)), ty::BoundRegion { var: ty::BoundVar::from(v), kind: ty::BoundRegionKind::Anon, @@ -1281,11 +1310,21 @@ impl<'tcx> CommonLifetimes<'tcx> { }) .collect(); + let anon_re_canonical_bounds = (0..NUM_PREINTERNED_ANON_RE_BOUNDS_V) + .map(|v| { + mk(ty::ReBound( + ty::BoundVarIndexKind::Canonical, + ty::BoundRegion { var: ty::BoundVar::from(v), kind: ty::BoundRegionKind::Anon }, + )) + }) + .collect(); + CommonLifetimes { re_static: mk(ty::ReStatic), re_erased: mk(ty::ReErased), re_vars, anon_re_bounds, + anon_re_canonical_bounds, } } } @@ -2012,7 +2051,7 @@ impl<'tcx> TyCtxt<'tcx> { if stable_crate_id == self.stable_crate_id(LOCAL_CRATE) { Some(self.untracked.definitions.read().local_def_path_hash_to_def_id(hash)?.to_def_id()) } else { - Some(self.def_path_hash_to_def_id_extern(hash, stable_crate_id)) + self.def_path_hash_to_def_id_extern(hash, stable_crate_id) } } @@ -2748,8 +2787,6 @@ direct_interners! { adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>, external_constraints: pub mk_external_constraints(ExternalConstraintsData<TyCtxt<'tcx>>): ExternalConstraints -> ExternalConstraints<'tcx>, - predefined_opaques_in_body: pub mk_predefined_opaques_in_body(PredefinedOpaquesData<TyCtxt<'tcx>>): - PredefinedOpaques -> PredefinedOpaques<'tcx>, } macro_rules! slice_interners { @@ -2786,6 +2823,7 @@ slice_interners!( offset_of: pub mk_offset_of((VariantIdx, FieldIdx)), patterns: pub mk_patterns(Pattern<'tcx>), outlives: pub mk_outlives(ty::ArgOutlivesPredicate<'tcx>), + predefined_opaques_in_body: pub mk_predefined_opaques_in_body((ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)), ); impl<'tcx> TyCtxt<'tcx> { @@ -3129,6 +3167,14 @@ impl<'tcx> TyCtxt<'tcx> { T::collect_and_apply(iter, |xs| self.mk_poly_existential_predicates(xs)) } + pub fn mk_predefined_opaques_in_body_from_iter<I, T>(self, iter: I) -> T::Output + where + I: Iterator<Item = T>, + T: CollectAndApply<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>), PredefinedOpaques<'tcx>>, + { + T::collect_and_apply(iter, |xs| self.mk_predefined_opaques_in_body(xs)) + } + pub fn mk_clauses_from_iter<I, T>(self, iter: I) -> T::Output where I: Iterator<Item = T>, diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs index 7d56ec1635f..ee29afcff63 100644 --- a/compiler/rustc_middle/src/ty/fold.rs +++ b/compiler/rustc_middle/src/ty/fold.rs @@ -125,7 +125,9 @@ where fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> { match *t.kind() { - ty::Bound(debruijn, bound_ty) if debruijn == self.current_index => { + ty::Bound(ty::BoundVarIndexKind::Bound(debruijn), bound_ty) + if debruijn == self.current_index => + { let ty = self.delegate.replace_ty(bound_ty); debug_assert!(!ty.has_vars_bound_above(ty::INNERMOST)); ty::shift_vars(self.tcx, ty, self.current_index.as_u32()) @@ -146,9 +148,11 @@ where fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> { match r.kind() { - ty::ReBound(debruijn, br) if debruijn == self.current_index => { + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), br) + if debruijn == self.current_index => + { let region = self.delegate.replace_region(br); - if let ty::ReBound(debruijn1, br) = region.kind() { + if let ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn1), br) = region.kind() { // If the callback returns a bound region, // that region should always use the INNERMOST // debruijn index. Then we adjust it to the @@ -165,7 +169,9 @@ where fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> { match ct.kind() { - ty::ConstKind::Bound(debruijn, bound_const) if debruijn == self.current_index => { + ty::ConstKind::Bound(ty::BoundVarIndexKind::Bound(debruijn), bound_const) + if debruijn == self.current_index => + { let ct = self.delegate.replace_const(bound_const); debug_assert!(!ct.has_vars_bound_above(ty::INNERMOST)); ty::shift_vars(self.tcx, ct, self.current_index.as_u32()) diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index c477e65f5d6..507a25a4325 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -16,7 +16,7 @@ use rustc_macros::{HashStable, TyDecodable, TyEncodable, extension}; use rustc_session::config::OptLevel; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, Symbol, sym}; use rustc_target::callconv::FnAbi; -use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, PanicStrategy, Target, X86Abi}; +use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, X86Abi}; use tracing::debug; use {rustc_abi as abi, rustc_hir as hir}; @@ -219,6 +219,15 @@ impl fmt::Display for ValidityRequirement { } #[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)] +pub enum SimdLayoutError { + /// The vector has 0 lanes. + ZeroLength, + /// The vector has more lanes than supported or permitted by + /// #\[rustc_simd_monomorphize_lane_limit\]. + TooManyLanes(u64), +} + +#[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)] pub enum LayoutError<'tcx> { /// A type doesn't have a sensible layout. /// @@ -230,6 +239,8 @@ pub enum LayoutError<'tcx> { Unknown(Ty<'tcx>), /// The size of a type exceeds [`TargetDataLayout::obj_size_bound`]. SizeOverflow(Ty<'tcx>), + /// A SIMD vector has invalid layout, such as zero-length or too many lanes. + InvalidSimd { ty: Ty<'tcx>, kind: SimdLayoutError }, /// The layout can vary due to a generic parameter. /// /// Unlike `Unknown`, this variant is a "soft" error and indicates that the layout @@ -257,6 +268,10 @@ impl<'tcx> LayoutError<'tcx> { match self { Unknown(_) => middle_layout_unknown, SizeOverflow(_) => middle_layout_size_overflow, + InvalidSimd { kind: SimdLayoutError::TooManyLanes(_), .. } => { + middle_layout_simd_too_many + } + InvalidSimd { kind: SimdLayoutError::ZeroLength, .. } => middle_layout_simd_zero_length, TooGeneric(_) => middle_layout_too_generic, NormalizationFailure(_, _) => middle_layout_normalization_failure, Cycle(_) => middle_layout_cycle, @@ -271,6 +286,10 @@ impl<'tcx> LayoutError<'tcx> { match self { Unknown(ty) => E::Unknown { ty }, SizeOverflow(ty) => E::Overflow { ty }, + InvalidSimd { ty, kind: SimdLayoutError::TooManyLanes(max_lanes) } => { + E::SimdTooManyLanes { ty, max_lanes } + } + InvalidSimd { ty, kind: SimdLayoutError::ZeroLength } => E::SimdZeroLength { ty }, TooGeneric(ty) => E::TooGeneric { ty }, NormalizationFailure(ty, e) => { E::NormalizationFailure { ty, failure_ty: e.get_type_for_failure() } @@ -293,6 +312,12 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> { LayoutError::SizeOverflow(ty) => { write!(f, "values of the type `{ty}` are too big for the target architecture") } + LayoutError::InvalidSimd { ty, kind: SimdLayoutError::TooManyLanes(max_lanes) } => { + write!(f, "the SIMD type `{ty}` has more elements than the limit {max_lanes}") + } + LayoutError::InvalidSimd { ty, kind: SimdLayoutError::ZeroLength } => { + write!(f, "the SIMD type `{ty}` has zero elements") + } LayoutError::NormalizationFailure(t, e) => write!( f, "unable to determine layout for `{}` because `{}` cannot be normalized", @@ -374,6 +399,7 @@ impl<'tcx> SizeSkeleton<'tcx> { e @ LayoutError::Cycle(_) | e @ LayoutError::Unknown(_) | e @ LayoutError::SizeOverflow(_) + | e @ LayoutError::InvalidSimd { .. } | e @ LayoutError::NormalizationFailure(..) | e @ LayoutError::ReferencesError(_), ) => return Err(e), @@ -1198,7 +1224,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi) // // Note that this is true regardless ABI specified on the function -- a `extern "C-unwind"` // function defined in Rust is also required to abort. - if tcx.sess.panic_strategy() == PanicStrategy::Abort && !tcx.is_foreign_item(did) { + if !tcx.sess.panic_strategy().unwinds() && !tcx.is_foreign_item(did) { return false; } @@ -1206,7 +1232,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi) // // This is not part of `codegen_fn_attrs` as it can differ between crates // and therefore cannot be computed in core. - if tcx.sess.opts.unstable_opts.panic_in_drop == PanicStrategy::Abort + if !tcx.sess.opts.unstable_opts.panic_in_drop.unwinds() && tcx.is_lang_item(did, LangItem::DropInPlace) { return false; @@ -1245,7 +1271,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi) | RiscvInterruptS | RustInvalid | Unadjusted => false, - Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind, + Rust | RustCall | RustCold => tcx.sess.panic_strategy().unwinds(), } } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 0ffef393a33..ce4de6b95e0 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -25,6 +25,7 @@ pub use generic_args::{GenericArgKind, TermKind, *}; pub use generics::*; pub use intrinsic::IntrinsicDef; use rustc_abi::{Align, FieldIdx, Integer, IntegerType, ReprFlags, ReprOptions, VariantIdx}; +use rustc_ast::expand::typetree::{FncTree, Kind, Type, TypeTree}; use rustc_ast::node_id::NodeMap; pub use rustc_ast_ir::{Movability, Mutability, try_visit}; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet}; @@ -62,7 +63,7 @@ pub use rustc_type_ir::solve::SizedTraitKind; pub use rustc_type_ir::*; #[allow(hidden_glob_reexports, unused_imports)] use rustc_type_ir::{InferCtxtLike, Interner}; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, trace}; pub use vtable::*; use {rustc_ast as ast, rustc_hir as hir}; @@ -2216,3 +2217,225 @@ pub struct DestructuredConst<'tcx> { pub variant: Option<VariantIdx>, pub fields: &'tcx [ty::Const<'tcx>], } + +/// Generate TypeTree information for autodiff. +/// This function creates TypeTree metadata that describes the memory layout +/// of function parameters and return types for Enzyme autodiff. +pub fn fnc_typetrees<'tcx>(tcx: TyCtxt<'tcx>, fn_ty: Ty<'tcx>) -> FncTree { + // Check if TypeTrees are disabled via NoTT flag + if tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::NoTT) { + return FncTree { args: vec![], ret: TypeTree::new() }; + } + + // Check if this is actually a function type + if !fn_ty.is_fn() { + return FncTree { args: vec![], ret: TypeTree::new() }; + } + + // Get the function signature + let fn_sig = fn_ty.fn_sig(tcx); + let sig = tcx.instantiate_bound_regions_with_erased(fn_sig); + + // Create TypeTrees for each input parameter + let mut args = vec![]; + for ty in sig.inputs().iter() { + let type_tree = typetree_from_ty(tcx, *ty); + args.push(type_tree); + } + + // Create TypeTree for return type + let ret = typetree_from_ty(tcx, sig.output()); + + FncTree { args, ret } +} + +/// Generate TypeTree for a specific type. +/// This function analyzes a Rust type and creates appropriate TypeTree metadata. +pub fn typetree_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> TypeTree { + let mut visited = Vec::new(); + typetree_from_ty_inner(tcx, ty, 0, &mut visited) +} + +/// Maximum recursion depth for TypeTree generation to prevent stack overflow +/// from pathological deeply nested types. Combined with cycle detection. +const MAX_TYPETREE_DEPTH: usize = 6; + +/// Internal recursive function for TypeTree generation with cycle detection and depth limiting. +fn typetree_from_ty_inner<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + depth: usize, + visited: &mut Vec<Ty<'tcx>>, +) -> TypeTree { + if depth >= MAX_TYPETREE_DEPTH { + trace!("typetree depth limit {} reached for type: {}", MAX_TYPETREE_DEPTH, ty); + return TypeTree::new(); + } + + if visited.contains(&ty) { + return TypeTree::new(); + } + + visited.push(ty); + let result = typetree_from_ty_impl(tcx, ty, depth, visited); + visited.pop(); + result +} + +/// Implementation of TypeTree generation logic. +fn typetree_from_ty_impl<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + depth: usize, + visited: &mut Vec<Ty<'tcx>>, +) -> TypeTree { + typetree_from_ty_impl_inner(tcx, ty, depth, visited, false) +} + +/// Internal implementation with context about whether this is for a reference target. +fn typetree_from_ty_impl_inner<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + depth: usize, + visited: &mut Vec<Ty<'tcx>>, + is_reference_target: bool, +) -> TypeTree { + if ty.is_scalar() { + let (kind, size) = if ty.is_integral() || ty.is_char() || ty.is_bool() { + (Kind::Integer, ty.primitive_size(tcx).bytes_usize()) + } else if ty.is_floating_point() { + match ty { + x if x == tcx.types.f16 => (Kind::Half, 2), + x if x == tcx.types.f32 => (Kind::Float, 4), + x if x == tcx.types.f64 => (Kind::Double, 8), + x if x == tcx.types.f128 => (Kind::F128, 16), + _ => (Kind::Integer, 0), + } + } else { + (Kind::Integer, 0) + }; + + // Use offset 0 for scalars that are direct targets of references (like &f64) + // Use offset -1 for scalars used directly (like function return types) + let offset = if is_reference_target && !ty.is_array() { 0 } else { -1 }; + return TypeTree(vec![Type { offset, size, kind, child: TypeTree::new() }]); + } + + if ty.is_ref() || ty.is_raw_ptr() || ty.is_box() { + let inner_ty = if let Some(inner) = ty.builtin_deref(true) { + inner + } else { + return TypeTree::new(); + }; + + let child = typetree_from_ty_impl_inner(tcx, inner_ty, depth + 1, visited, true); + return TypeTree(vec![Type { + offset: -1, + size: tcx.data_layout.pointer_size().bytes_usize(), + kind: Kind::Pointer, + child, + }]); + } + + if ty.is_array() { + if let ty::Array(element_ty, len_const) = ty.kind() { + let len = len_const.try_to_target_usize(tcx).unwrap_or(0); + if len == 0 { + return TypeTree::new(); + } + let element_tree = + typetree_from_ty_impl_inner(tcx, *element_ty, depth + 1, visited, false); + let mut types = Vec::new(); + for elem_type in &element_tree.0 { + types.push(Type { + offset: -1, + size: elem_type.size, + kind: elem_type.kind, + child: elem_type.child.clone(), + }); + } + + return TypeTree(types); + } + } + + if ty.is_slice() { + if let ty::Slice(element_ty) = ty.kind() { + let element_tree = + typetree_from_ty_impl_inner(tcx, *element_ty, depth + 1, visited, false); + return element_tree; + } + } + + if let ty::Tuple(tuple_types) = ty.kind() { + if tuple_types.is_empty() { + return TypeTree::new(); + } + + let mut types = Vec::new(); + let mut current_offset = 0; + + for tuple_ty in tuple_types.iter() { + let element_tree = + typetree_from_ty_impl_inner(tcx, tuple_ty, depth + 1, visited, false); + + let element_layout = tcx + .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(tuple_ty)) + .ok() + .map(|layout| layout.size.bytes_usize()) + .unwrap_or(0); + + for elem_type in &element_tree.0 { + types.push(Type { + offset: if elem_type.offset == -1 { + current_offset as isize + } else { + current_offset as isize + elem_type.offset + }, + size: elem_type.size, + kind: elem_type.kind, + child: elem_type.child.clone(), + }); + } + + current_offset += element_layout; + } + + return TypeTree(types); + } + + if let ty::Adt(adt_def, args) = ty.kind() { + if adt_def.is_struct() { + let struct_layout = + tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)); + if let Ok(layout) = struct_layout { + let mut types = Vec::new(); + + for (field_idx, field_def) in adt_def.all_fields().enumerate() { + let field_ty = field_def.ty(tcx, args); + let field_tree = + typetree_from_ty_impl_inner(tcx, field_ty, depth + 1, visited, false); + + let field_offset = layout.fields.offset(field_idx).bytes_usize(); + + for elem_type in &field_tree.0 { + types.push(Type { + offset: if elem_type.offset == -1 { + field_offset as isize + } else { + field_offset as isize + elem_type.offset + }, + size: elem_type.size, + kind: elem_type.kind, + child: elem_type.child.clone(), + }); + } + } + + return TypeTree(types); + } + } + } + + TypeTree::new() +} diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index 1b7ef8de845..4d1fcaeda5e 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -2664,7 +2664,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> { fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> { let name = &mut self.name; let region = match r.kind() { - ty::ReBound(db, br) if db >= self.current_index => { + ty::ReBound(ty::BoundVarIndexKind::Bound(db), br) if db >= self.current_index => { *self.region_map.entry(br).or_insert_with(|| name(Some(db), self.current_index, br)) } ty::RePlaceholder(ty::PlaceholderRegion { @@ -2687,7 +2687,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> { } _ => return r, }; - if let ty::ReBound(debruijn1, br) = region.kind() { + if let ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn1), br) = region.kind() { assert_eq!(debruijn1, ty::INNERMOST); ty::Region::new_bound(self.tcx, self.current_index, br) } else { @@ -3177,8 +3177,7 @@ define_print! { write!(p, "` can be evaluated")?; } ty::ClauseKind::UnstableFeature(symbol) => { - write!(p, "unstable feature: ")?; - write!(p, "`{symbol}`")?; + write!(p, "feature({symbol}) is enabled")?; } } } diff --git a/compiler/rustc_middle/src/ty/region.rs b/compiler/rustc_middle/src/ty/region.rs index 3a7852dea06..f0687f2bc72 100644 --- a/compiler/rustc_middle/src/ty/region.rs +++ b/compiler/rustc_middle/src/ty/region.rs @@ -31,7 +31,7 @@ impl<'tcx> rustc_type_ir::Flags for Region<'tcx> { fn outer_exclusive_binder(&self) -> ty::DebruijnIndex { match self.kind() { - ty::ReBound(debruijn, _) => debruijn.shifted_in(1), + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), _) => debruijn.shifted_in(1), _ => ty::INNERMOST, } } @@ -59,7 +59,20 @@ impl<'tcx> Region<'tcx> { { re } else { - tcx.intern_region(ty::ReBound(debruijn, bound_region)) + tcx.intern_region(ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), bound_region)) + } + } + + #[inline] + pub fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: ty::BoundVar) -> Region<'tcx> { + // Use a pre-interned one when possible. + if let Some(re) = tcx.lifetimes.anon_re_canonical_bounds.get(var.as_usize()).copied() { + re + } else { + tcx.intern_region(ty::ReBound( + ty::BoundVarIndexKind::Canonical, + ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon }, + )) } } @@ -122,7 +135,12 @@ impl<'tcx> Region<'tcx> { pub fn new_from_kind(tcx: TyCtxt<'tcx>, kind: RegionKind<'tcx>) -> Region<'tcx> { match kind { ty::ReEarlyParam(region) => Region::new_early_param(tcx, region), - ty::ReBound(debruijn, region) => Region::new_bound(tcx, debruijn, region), + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), region) => { + Region::new_bound(tcx, debruijn, region) + } + ty::ReBound(ty::BoundVarIndexKind::Canonical, region) => { + Region::new_canonical_bound(tcx, region.var) + } ty::ReLateParam(ty::LateParamRegion { scope, kind }) => { Region::new_late_param(tcx, scope, kind) } @@ -148,6 +166,10 @@ impl<'tcx> rustc_type_ir::inherent::Region<TyCtxt<'tcx>> for Region<'tcx> { Region::new_bound(tcx, debruijn, ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon }) } + fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: rustc_type_ir::BoundVar) -> Self { + Region::new_canonical_bound(tcx, var) + } + fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderRegion) -> Self { Region::new_placeholder(tcx, placeholder) } @@ -223,7 +245,7 @@ impl<'tcx> Region<'tcx> { #[inline] pub fn bound_at_or_above_binder(self, index: ty::DebruijnIndex) -> bool { match self.kind() { - ty::ReBound(debruijn, _) => debruijn >= index, + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), _) => debruijn >= index, _ => false, } } @@ -254,7 +276,11 @@ impl<'tcx> Region<'tcx> { ty::ReStatic => { flags = flags | TypeFlags::HAS_FREE_REGIONS; } - ty::ReBound(..) => { + ty::ReBound(ty::BoundVarIndexKind::Canonical, _) => { + flags = flags | TypeFlags::HAS_RE_BOUND; + flags = flags | TypeFlags::HAS_CANONICAL_BOUND; + } + ty::ReBound(ty::BoundVarIndexKind::Bound(..), _) => { flags = flags | TypeFlags::HAS_RE_BOUND; } ty::ReErased => { diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 11d109b463d..8dc4dfd677b 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -796,6 +796,7 @@ macro_rules! list_fold { list_fold! { &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> : mk_poly_existential_predicates, + &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>: mk_predefined_opaques_in_body, &'tcx ty::List<PlaceElem<'tcx>> : mk_place_elems, &'tcx ty::List<ty::Pattern<'tcx>> : mk_patterns, &'tcx ty::List<ty::ArgOutlivesPredicate<'tcx>> : mk_outlives, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index de35e5e847c..a3fdd4e35b6 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -487,7 +487,23 @@ impl<'tcx> Ty<'tcx> { { ty } else { - Ty::new(tcx, Bound(index, bound_ty)) + Ty::new(tcx, Bound(ty::BoundVarIndexKind::Bound(index), bound_ty)) + } + } + + #[inline] + pub fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: BoundVar) -> Ty<'tcx> { + // Use a pre-interned one when possible. + if let Some(ty) = tcx.types.anon_canonical_bound_tys.get(var.as_usize()).copied() { + ty + } else { + Ty::new( + tcx, + Bound( + ty::BoundVarIndexKind::Canonical, + ty::BoundTy { var, kind: ty::BoundTyKind::Anon }, + ), + ) } } @@ -952,6 +968,10 @@ impl<'tcx> rustc_type_ir::inherent::Ty<TyCtxt<'tcx>> for Ty<'tcx> { Ty::new_bound(tcx, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon }) } + fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: ty::BoundVar) -> Self { + Ty::new_canonical_bound(tcx, var) + } + fn new_alias( interner: TyCtxt<'tcx>, kind: ty::AliasTyKind, diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs index 8dd80aab946..b276b993ec9 100644 --- a/compiler/rustc_middle/src/ty/typeck_results.rs +++ b/compiler/rustc_middle/src/ty/typeck_results.rs @@ -167,7 +167,7 @@ pub struct TypeckResults<'tcx> { /// We also store the type here, so that the compiler can use it as a hint /// for figuring out hidden types, even if they are only set in dead code /// (which doesn't show up in MIR). - pub concrete_opaque_types: FxIndexMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>, + pub hidden_types: FxIndexMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>, /// Tracks the minimum captures required for a closure; /// see `MinCaptureInformationMap` for more details. @@ -250,7 +250,7 @@ impl<'tcx> TypeckResults<'tcx> { coercion_casts: Default::default(), used_trait_imports: Default::default(), tainted_by_errors: None, - concrete_opaque_types: Default::default(), + hidden_types: Default::default(), closure_min_captures: Default::default(), closure_fake_reads: Default::default(), rvalue_scopes: Default::default(), @@ -798,8 +798,8 @@ impl<'tcx> IsIdentity for CanonicalUserType<'tcx> { match arg.kind() { GenericArgKind::Type(ty) => match ty.kind() { ty::Bound(debruijn, b) => { - // We only allow a `ty::INNERMOST` index in generic parameters. - assert_eq!(*debruijn, ty::INNERMOST); + // We only allow a `ty::BoundVarIndexKind::Canonical` index in generic parameters. + assert_eq!(*debruijn, ty::BoundVarIndexKind::Canonical); cvar == b.var } _ => false, @@ -807,8 +807,8 @@ impl<'tcx> IsIdentity for CanonicalUserType<'tcx> { GenericArgKind::Lifetime(r) => match r.kind() { ty::ReBound(debruijn, b) => { - // We only allow a `ty::INNERMOST` index in generic parameters. - assert_eq!(debruijn, ty::INNERMOST); + // We only allow a `ty::BoundVarIndexKind::Canonical` index in generic parameters. + assert_eq!(debruijn, ty::BoundVarIndexKind::Canonical); cvar == b.var } _ => false, @@ -816,8 +816,8 @@ impl<'tcx> IsIdentity for CanonicalUserType<'tcx> { GenericArgKind::Const(ct) => match ct.kind() { ty::ConstKind::Bound(debruijn, b) => { - // We only allow a `ty::INNERMOST` index in generic parameters. - assert_eq!(debruijn, ty::INNERMOST); + // We only allow a `ty::BoundVarIndexKind::Canonical` index in generic parameters. + assert_eq!(debruijn, ty::BoundVarIndexKind::Canonical); cvar == b.var } _ => false, diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 4f039381e50..a4422abc688 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -1368,7 +1368,6 @@ impl<'tcx> Ty<'tcx> { /// 2229 drop reorder migration analysis. #[inline] pub fn has_significant_drop(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool { - assert!(!self.has_non_region_infer()); // Avoid querying in simple cases. match needs_drop_components(tcx, self) { Err(AlwaysRequiresDrop) => true, @@ -1381,6 +1380,16 @@ impl<'tcx> Ty<'tcx> { _ => self, }; + // FIXME + // We should be canonicalizing, or else moving this to a method of inference + // context, or *something* like that, + // but for now just avoid passing inference variables + // to queries that can't cope with them. + // Instead, conservatively return "true" (may change drop order). + if query_ty.has_infer() { + return true; + } + // This doesn't depend on regions, so try to minimize distinct // query keys used. let erased = tcx.normalize_erasing_regions(typing_env, query_ty); diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs index f0c47f257cc..e84ac56b31d 100644 --- a/compiler/rustc_middle/src/ty/visit.rs +++ b/compiler/rustc_middle/src/ty/visit.rs @@ -78,7 +78,9 @@ impl<'tcx> TyCtxt<'tcx> { fn visit_region(&mut self, r: ty::Region<'tcx>) -> Self::Result { match r.kind() { - ty::ReBound(debruijn, _) if debruijn < self.outer_index => { + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), _) + if debruijn < self.outer_index => + { ControlFlow::Continue(()) } _ => { @@ -205,7 +207,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for LateBoundRegionsCollector { } fn visit_region(&mut self, r: ty::Region<'tcx>) { - if let ty::ReBound(debruijn, br) = r.kind() { + if let ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), br) = r.kind() { if debruijn == self.current_index { self.regions.insert(br.kind); } diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs index e2f09fdcb4b..a3e9054fdcb 100644 --- a/compiler/rustc_middle/src/ty/vtable.rs +++ b/compiler/rustc_middle/src/ty/vtable.rs @@ -104,7 +104,7 @@ pub(super) fn vtable_allocation_provider<'tcx>( .expect("failed to build vtable representation"); assert!(layout.is_sized(), "can't create a vtable for an unsized type"); let size = layout.size.bytes(); - let align = layout.align.abi.bytes(); + let align = layout.align.bytes(); let ptr_size = tcx.data_layout.pointer_size(); let ptr_align = tcx.data_layout.pointer_align().abi; |