diff options
Diffstat (limited to 'compiler')
378 files changed, 24288 insertions, 6256 deletions
diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs index 34736b820e9..b76e1e7ce65 100644 --- a/compiler/rustc_arena/src/lib.rs +++ b/compiler/rustc_arena/src/lib.rs @@ -217,14 +217,18 @@ impl<T> TypedArena<T> { let mut chunks = self.chunks.borrow_mut(); let mut new_cap; if let Some(last_chunk) = chunks.last_mut() { - let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; - last_chunk.entries = used_bytes / mem::size_of::<T>(); + // If a type is `!needs_drop`, we don't need to keep track of how many elements + // the chunk stores - the field will be ignored anyway. + if mem::needs_drop::<T>() { + let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize; + last_chunk.entries = used_bytes / mem::size_of::<T>(); + } // If the previous chunk's len is less than HUGE_PAGE // bytes, then this chunk will be least double the previous // chunk's size. new_cap = last_chunk.storage.len().min(HUGE_PAGE / elem_size / 2); - new_cap = new_cap * 2; + new_cap *= 2; } else { new_cap = PAGE / elem_size; } @@ -342,7 +346,7 @@ impl DroplessArena { // bytes, then this chunk will be least double the previous // chunk's size. new_cap = last_chunk.storage.len().min(HUGE_PAGE / 2); - new_cap = new_cap * 2; + new_cap *= 2; } else { new_cap = PAGE; } @@ -529,7 +533,7 @@ impl DropArena { ptr::write(mem, object); let result = &mut *mem; // Record the destructor after doing the allocation as that may panic - // and would cause `object`'s destuctor to run twice if it was recorded before + // and would cause `object`'s destructor to run twice if it was recorded before self.destructors .borrow_mut() .push(DropType { drop_fn: drop_for_type::<T>, obj: result as *mut T as *mut u8 }); @@ -556,12 +560,10 @@ impl DropArena { mem::forget(vec.drain(..)); // Record the destructors after doing the allocation as that may panic - // and would cause `object`'s destuctor to run twice if it was recorded before + // and would cause `object`'s destructor to run twice if it was recorded before for i in 0..len { - destructors.push(DropType { - drop_fn: drop_for_type::<T>, - obj: start_ptr.offset(i as isize) as *mut u8, - }); + destructors + .push(DropType { drop_fn: drop_for_type::<T>, obj: start_ptr.add(i) as *mut u8 }); } slice::from_raw_parts_mut(start_ptr, len) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index ea84fc0095f..7d5e235c885 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -24,7 +24,7 @@ pub use UnsafeSource::*; use crate::ptr::P; use crate::token::{self, CommentKind, DelimToken}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; +use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::stack::ensure_sufficient_stack; @@ -97,7 +97,7 @@ pub struct Path { /// The segments in the path: the things separated by `::`. /// Global paths begin with `kw::PathRoot`. pub segments: Vec<PathSegment>, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } impl PartialEq<Symbol> for Path { @@ -222,6 +222,15 @@ pub enum AngleBracketedArg { Constraint(AssocTyConstraint), } +impl AngleBracketedArg { + pub fn span(&self) -> Span { + match self { + AngleBracketedArg::Arg(arg) => arg.span(), + AngleBracketedArg::Constraint(constraint) => constraint.span, + } + } +} + impl Into<Option<P<GenericArgs>>> for AngleBracketedArgs { fn into(self) -> Option<P<GenericArgs>> { Some(P(GenericArgs::AngleBracketed(self))) @@ -535,7 +544,7 @@ pub struct Block { /// Distinguishes between `unsafe { ... }` and `{ ... }`. pub rules: BlockCheckMode, pub span: Span, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } /// A match pattern. @@ -546,7 +555,7 @@ pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } impl Pat { @@ -892,7 +901,7 @@ pub struct Stmt { pub id: NodeId, pub kind: StmtKind, pub span: Span, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } impl Stmt { @@ -1040,7 +1049,7 @@ pub struct Expr { pub kind: ExprKind, pub span: Span, pub attrs: AttrVec, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -1835,7 +1844,7 @@ pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } impl Clone for Ty { @@ -2408,7 +2417,7 @@ impl<D: Decoder> rustc_serialize::Decodable<D> for AttrId { pub struct AttrItem { pub path: Path, pub args: MacArgs, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } /// A list of attributes. @@ -2423,6 +2432,7 @@ pub struct Attribute { /// or the construct this attribute is contained within (inner). pub style: AttrStyle, pub span: Span, + pub tokens: Option<LazyTokenStream>, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2482,7 +2492,7 @@ pub enum CrateSugar { pub struct Visibility { pub kind: VisibilityKind, pub span: Span, - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2569,7 +2579,7 @@ pub struct Item<K = ItemKind> { /// /// Note that the tokens here do not include the outer attributes, but will /// include inner attributes. - pub tokens: Option<TokenStream>, + pub tokens: Option<LazyTokenStream>, } impl Item { diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 8351be222f6..70ad43ecad2 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -325,7 +325,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri } pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute { - Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span } + Attribute { kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, tokens: None } } /// Returns an inner attribute with the given value and span. @@ -344,7 +344,13 @@ pub fn mk_doc_comment( data: Symbol, span: Span, ) -> Attribute { - Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span } + Attribute { + kind: AttrKind::DocComment(comment_kind, data), + id: mk_attr_id(), + style, + span, + tokens: None, + } } pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { @@ -623,7 +629,8 @@ impl HasAttrs for StmtKind { match *self { StmtKind::Local(ref local) => local.attrs(), StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(), - StmtKind::Empty | StmtKind::Item(..) => &[], + StmtKind::Item(ref item) => item.attrs(), + StmtKind::Empty => &[], StmtKind::MacCall(ref mac) => mac.attrs.attrs(), } } @@ -632,7 +639,8 @@ impl HasAttrs for StmtKind { match self { StmtKind::Local(local) => local.visit_attrs(f), StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.visit_attrs(f), - StmtKind::Empty | StmtKind::Item(..) => {} + StmtKind::Item(item) => item.visit_attrs(f), + StmtKind::Empty => {} StmtKind::MacCall(mac) => { mac.attrs.visit_attrs(f); } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 382003c834e..166d36ce424 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -577,7 +577,7 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) { } pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) { - let Attribute { kind, id: _, style: _, span } = attr; + let Attribute { kind, id: _, style: _, span, tokens: _ } = attr; match kind { AttrKind::Normal(AttrItem { path, args, tokens: _ }) => { vis.visit_path(path); diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index d991027cb45..2bba7e618c0 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -303,6 +303,13 @@ impl TokenKind { _ => None, } } + + pub fn should_end_const_arg(&self) -> bool { + match self { + Gt | Ge | BinOp(Shr) | BinOpEq(Shr) => true, + _ => false, + } + } } impl Token { diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 8acb6b2f375..98da20af8f6 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -16,12 +16,13 @@ use crate::token::{self, DelimToken, Token, TokenKind}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_data_structures::sync::Lrc; +use rustc_data_structures::sync::{self, Lrc}; use rustc_macros::HashStable_Generic; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_span::{Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; -use std::{iter, mem}; +use std::{fmt, iter, mem}; /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very @@ -119,13 +120,64 @@ where } } +pub trait CreateTokenStream: sync::Send + sync::Sync { + fn create_token_stream(&self) -> TokenStream; +} + +impl CreateTokenStream for TokenStream { + fn create_token_stream(&self) -> TokenStream { + self.clone() + } +} + +/// A lazy version of `TokenStream`, which defers creation +/// of an actual `TokenStream` until it is needed. +/// `Box` is here only to reduce the structure size. +#[derive(Clone)] +pub struct LazyTokenStream(Lrc<Box<dyn CreateTokenStream>>); + +impl LazyTokenStream { + pub fn new(inner: impl CreateTokenStream + 'static) -> LazyTokenStream { + LazyTokenStream(Lrc::new(Box::new(inner))) + } + + pub fn create_token_stream(&self) -> TokenStream { + self.0.create_token_stream() + } +} + +impl fmt::Debug for LazyTokenStream { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt("LazyTokenStream", f) + } +} + +impl<S: Encoder> Encodable<S> for LazyTokenStream { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + // Used by AST json printing. + Encodable::encode(&self.create_token_stream(), s) + } +} + +impl<D: Decoder> Decodable<D> for LazyTokenStream { + fn decode(_d: &mut D) -> Result<Self, D::Error> { + panic!("Attempted to decode LazyTokenStream"); + } +} + +impl<CTX> HashStable<CTX> for LazyTokenStream { + fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { + panic!("Attempted to compute stable hash for LazyTokenStream"); + } +} + /// A `TokenStream` is an abstract sequence of tokens, organized into `TokenTree`s. /// /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat. #[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct TokenStream(pub Lrc<Vec<TreeAndSpacing>>); +pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>); pub type TreeAndSpacing = (TokenTree, Spacing); @@ -286,12 +338,12 @@ impl TokenStream { t1.next().is_none() && t2.next().is_none() } - pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { + pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { TokenStream(Lrc::new( self.0 .iter() .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) + .map(|(i, (tree, is_joint))| (f(i, tree), *is_joint)) .collect(), )) } @@ -394,8 +446,8 @@ impl Cursor { self.index = index; } - pub fn look_ahead(&self, n: usize) -> Option<TokenTree> { - self.stream.0[self.index..].get(n).map(|(tree, _)| tree.clone()) + pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { + self.stream.0[self.index..].get(n).map(|(tree, _)| tree) } } diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index c49fd76a313..a6ac056b93b 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -210,9 +210,9 @@ impl<'hir> LoweringContext<'_, 'hir> { ex.span = e.span; } // Merge attributes into the inner expression. - let mut attrs = e.attrs.clone(); + let mut attrs: Vec<_> = e.attrs.iter().map(|a| self.lower_attr(a)).collect(); attrs.extend::<Vec<_>>(ex.attrs.into()); - ex.attrs = attrs; + ex.attrs = attrs.into(); return ex; } @@ -1471,13 +1471,15 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::MatchSource::ForLoopDesugar, )); + let attrs: Vec<_> = e.attrs.iter().map(|a| self.lower_attr(a)).collect(); + // This is effectively `{ let _result = ...; _result }`. // The construct was introduced in #21984 and is necessary to make sure that // temporaries in the `head` expression are dropped and do not leak to the // surrounding scope of the `match` since the `match` is not a terminating scope. // // Also, add the attributes to the outer returned expr node. - self.expr_drop_temps_mut(desugared_span, match_expr, e.attrs.clone()) + self.expr_drop_temps_mut(desugared_span, match_expr, attrs.into()) } /// Desugar `ExprKind::Try` from: `<expr>?` into: diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index a28d022c661..599599f415f 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -148,7 +148,7 @@ struct LoweringContext<'a, 'hir: 'a> { is_collecting_in_band_lifetimes: bool, /// Currently in-scope lifetimes defined in impl headers, fn headers, or HRTB. - /// When `is_collectin_in_band_lifetimes` is true, each lifetime is checked + /// When `is_collecting_in_band_lifetimes` is true, each lifetime is checked /// against this list to see if it is already in-scope, or if a definition /// needs to be created for it. /// @@ -257,7 +257,7 @@ enum ImplTraitPosition { /// Disallowed in `let` / `const` / `static` bindings. Binding, - /// All other posiitons. + /// All other positions. Other, } @@ -363,7 +363,7 @@ enum ParenthesizedGenericArgs { /// elided bounds follow special rules. Note that this only covers /// cases where *nothing* is written; the `'_` in `Box<dyn Foo + /// '_>` is a case of "modern" elision. -/// - **Deprecated** -- this coverse cases like `Ref<T>`, where the lifetime +/// - **Deprecated** -- this covers cases like `Ref<T>`, where the lifetime /// parameter to ref is completely elided. `Ref<'_, T>` would be the modern, /// non-deprecated equivalent. /// @@ -490,10 +490,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let count = generics .params .iter() - .filter(|param| match param.kind { - ast::GenericParamKind::Lifetime { .. } => true, - _ => false, - }) + .filter(|param| matches!(param.kind, ast::GenericParamKind::Lifetime { .. })) .count(); self.lctx.type_def_lifetime_params.insert(def_id.to_def_id(), count); } @@ -538,6 +535,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } self.visit_fn_ret_ty(&f.decl.output) } + TyKind::ImplTrait(def_node_id, _) => { + self.lctx.allocate_hir_id_counter(def_node_id); + self.with_hir_id_owner(Some(def_node_id), |this| { + visit::walk_ty(this, t); + }); + } _ => visit::walk_ty(self, t), } } @@ -972,7 +975,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data), }; - Attribute { kind, id: attr.id, style: attr.style, span: attr.span } + // Tokens aren't needed after macro expansion and parsing + Attribute { kind, id: attr.id, style: attr.style, span: attr.span, tokens: None } } fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs { @@ -1346,10 +1350,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // Add a definition for the in-band `Param`. let def_id = self.resolver.local_def_id(def_node_id); - let hir_bounds = self.lower_param_bounds( - bounds, - ImplTraitContext::Universal(in_band_ty_params), - ); + self.allocate_hir_id_counter(def_node_id); + + let hir_bounds = self.with_hir_id_owner(def_node_id, |this| { + this.lower_param_bounds( + bounds, + ImplTraitContext::Universal(in_band_ty_params), + ) + }); // Set the name to `impl Bound1 + Bound2`. let ident = Ident::from_str_and_span(&pprust::ty_to_string(t), span); in_band_ty_params.push(hir::GenericParam { @@ -1713,7 +1721,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { pat: self.lower_pat(&l.pat), init, span: l.span, - attrs: l.attrs.clone(), + attrs: l.attrs.iter().map(|a| self.lower_attr(a)).collect::<Vec<_>>().into(), source: hir::LocalSource::Normal, }, ids, @@ -2200,7 +2208,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { .attrs .iter() .filter(|attr| self.sess.check_name(attr, sym::rustc_synthetic)) - .map(|_| hir::SyntheticTyParamKind::ImplTrait) + .map(|_| hir::SyntheticTyParamKind::FromAttr) .next(), }; diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs index cb7b7c0eb61..a1cbcde1f42 100644 --- a/compiler/rustc_ast_lowering/src/pat.rs +++ b/compiler/rustc_ast_lowering/src/pat.rs @@ -10,82 +10,90 @@ use rustc_span::symbol::Ident; use rustc_span::{source_map::Spanned, Span}; impl<'a, 'hir> LoweringContext<'a, 'hir> { - crate fn lower_pat(&mut self, p: &Pat) -> &'hir hir::Pat<'hir> { + crate fn lower_pat(&mut self, mut pattern: &Pat) -> &'hir hir::Pat<'hir> { ensure_sufficient_stack(|| { - let node = match p.kind { - PatKind::Wild => hir::PatKind::Wild, - PatKind::Ident(ref binding_mode, ident, ref sub) => { - let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(&*s)); - let node = self.lower_pat_ident(p, binding_mode, ident, lower_sub); - node - } - PatKind::Lit(ref e) => hir::PatKind::Lit(self.lower_expr(e)), - PatKind::TupleStruct(ref path, ref pats) => { - let qpath = self.lower_qpath( - p.id, - &None, - path, - ParamMode::Optional, - ImplTraitContext::disallowed(), - ); - let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple struct"); - hir::PatKind::TupleStruct(qpath, pats, ddpos) - } - PatKind::Or(ref pats) => hir::PatKind::Or( - self.arena.alloc_from_iter(pats.iter().map(|x| self.lower_pat(x))), - ), - PatKind::Path(ref qself, ref path) => { - let qpath = self.lower_qpath( - p.id, - qself, - path, - ParamMode::Optional, - ImplTraitContext::disallowed(), - ); - hir::PatKind::Path(qpath) - } - PatKind::Struct(ref path, ref fields, etc) => { - let qpath = self.lower_qpath( - p.id, - &None, - path, - ParamMode::Optional, - ImplTraitContext::disallowed(), - ); + // loop here to avoid recursion + let node = loop { + match pattern.kind { + PatKind::Wild => break hir::PatKind::Wild, + PatKind::Ident(ref binding_mode, ident, ref sub) => { + let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(&*s)); + break self.lower_pat_ident(pattern, binding_mode, ident, lower_sub); + } + PatKind::Lit(ref e) => break hir::PatKind::Lit(self.lower_expr(e)), + PatKind::TupleStruct(ref path, ref pats) => { + let qpath = self.lower_qpath( + pattern.id, + &None, + path, + ParamMode::Optional, + ImplTraitContext::disallowed(), + ); + let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple struct"); + break hir::PatKind::TupleStruct(qpath, pats, ddpos); + } + PatKind::Or(ref pats) => { + break hir::PatKind::Or( + self.arena.alloc_from_iter(pats.iter().map(|x| self.lower_pat(x))), + ); + } + PatKind::Path(ref qself, ref path) => { + let qpath = self.lower_qpath( + pattern.id, + qself, + path, + ParamMode::Optional, + ImplTraitContext::disallowed(), + ); + break hir::PatKind::Path(qpath); + } + PatKind::Struct(ref path, ref fields, etc) => { + let qpath = self.lower_qpath( + pattern.id, + &None, + path, + ParamMode::Optional, + ImplTraitContext::disallowed(), + ); - let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::FieldPat { - hir_id: self.next_id(), - ident: f.ident, - pat: self.lower_pat(&f.pat), - is_shorthand: f.is_shorthand, - span: f.span, - })); - hir::PatKind::Struct(qpath, fs, etc) - } - PatKind::Tuple(ref pats) => { - let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple"); - hir::PatKind::Tuple(pats, ddpos) - } - PatKind::Box(ref inner) => hir::PatKind::Box(self.lower_pat(inner)), - PatKind::Ref(ref inner, mutbl) => hir::PatKind::Ref(self.lower_pat(inner), mutbl), - PatKind::Range(ref e1, ref e2, Spanned { node: ref end, .. }) => { - hir::PatKind::Range( - e1.as_deref().map(|e| self.lower_expr(e)), - e2.as_deref().map(|e| self.lower_expr(e)), - self.lower_range_end(end, e2.is_some()), - ) - } - PatKind::Slice(ref pats) => self.lower_pat_slice(pats), - PatKind::Rest => { - // If we reach here the `..` pattern is not semantically allowed. - self.ban_illegal_rest_pat(p.span) + let fs = self.arena.alloc_from_iter(fields.iter().map(|f| hir::FieldPat { + hir_id: self.next_id(), + ident: f.ident, + pat: self.lower_pat(&f.pat), + is_shorthand: f.is_shorthand, + span: f.span, + })); + break hir::PatKind::Struct(qpath, fs, etc); + } + PatKind::Tuple(ref pats) => { + let (pats, ddpos) = self.lower_pat_tuple(pats, "tuple"); + break hir::PatKind::Tuple(pats, ddpos); + } + PatKind::Box(ref inner) => { + break hir::PatKind::Box(self.lower_pat(inner)); + } + PatKind::Ref(ref inner, mutbl) => { + break hir::PatKind::Ref(self.lower_pat(inner), mutbl); + } + PatKind::Range(ref e1, ref e2, Spanned { node: ref end, .. }) => { + break hir::PatKind::Range( + e1.as_deref().map(|e| self.lower_expr(e)), + e2.as_deref().map(|e| self.lower_expr(e)), + self.lower_range_end(end, e2.is_some()), + ); + } + PatKind::Slice(ref pats) => break self.lower_pat_slice(pats), + PatKind::Rest => { + // If we reach here the `..` pattern is not semantically allowed. + break self.ban_illegal_rest_pat(pattern.span); + } + // return inner to be processed in next loop + PatKind::Paren(ref inner) => pattern = inner, + PatKind::MacCall(_) => panic!("{:?} shouldn't exist here", pattern.span), } - // FIXME: consider not using recursion to lower this. - PatKind::Paren(ref inner) => return self.lower_pat(inner), - PatKind::MacCall(_) => panic!("{:?} shouldn't exist here", p.span), }; - self.pat_with_node_id_of(p, node) + self.pat_with_node_id_of(pattern, node) }) } diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs index cf68dfb9ae1..6afed355dc3 100644 --- a/compiler/rustc_ast_lowering/src/path.rs +++ b/compiler/rustc_ast_lowering/src/path.rs @@ -262,10 +262,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { self.lower_angle_bracketed_parameter_data(&Default::default(), param_mode, itctx) }; - let has_lifetimes = generic_args.args.iter().any(|arg| match arg { - GenericArg::Lifetime(_) => true, - _ => false, - }); + let has_lifetimes = + generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_))); let first_generic_span = generic_args .args .iter() @@ -310,8 +308,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { E0726, "implicit elided lifetime not allowed here" ); - rustc_session::lint::add_elided_lifetime_in_path_suggestion( - &self.sess, + rustc_errors::add_elided_lifetime_in_path_suggestion( + &self.sess.source_map(), &mut err, expected_lifetimes, path_span, diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs index b1c8e0ee727..d6585bcc425 100644 --- a/compiler/rustc_ast_passes/src/ast_validation.rs +++ b/compiler/rustc_ast_passes/src/ast_validation.rs @@ -516,7 +516,7 @@ impl<'a> AstValidator<'a> { self.session.source_map().guess_head_span(self.extern_mod.unwrap().span) } - /// An `fn` in `extern { ... }` cannot have qualfiers, e.g. `async fn`. + /// An `fn` in `extern { ... }` cannot have qualifiers, e.g. `async fn`. fn check_foreign_fn_headerless(&self, ident: Ident, span: Span, header: FnHeader) { if header.has_qualifiers() { self.err_handler() diff --git a/compiler/rustc_ast_pretty/src/pp.rs b/compiler/rustc_ast_pretty/src/pp.rs index 95f969d7691..56e769ba6b7 100644 --- a/compiler/rustc_ast_pretty/src/pp.rs +++ b/compiler/rustc_ast_pretty/src/pp.rs @@ -390,7 +390,7 @@ impl Printer { self.scan_stack.pop_front().unwrap() } - fn scan_top(&mut self) -> usize { + fn scan_top(&self) -> usize { *self.scan_stack.front().unwrap() } @@ -484,7 +484,7 @@ impl Printer { self.pending_indentation += amount; } - fn get_top(&mut self) -> PrintStackElem { + fn get_top(&self) -> PrintStackElem { *self.print_stack.last().unwrap_or({ &PrintStackElem { offset: 0, pbreak: PrintStackBreak::Broken(Breaks::Inconsistent) } }) diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index af8f8132780..9fcba902443 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -63,7 +63,7 @@ impl<'a> Comments<'a> { } pub fn trailing_comment( - &mut self, + &self, span: rustc_span::Span, next_pos: Option<BytePos>, ) -> Option<Comment> { @@ -156,24 +156,13 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { } } match tt { - TokenTree::Token(token) => match token.kind { - token::Comma => false, - _ => true, - }, - TokenTree::Delimited(_, DelimToken::Paren, _) => match prev { - TokenTree::Token(token) => match token.kind { - token::Ident(_, _) => false, - _ => true, - }, - _ => true, - }, - TokenTree::Delimited(_, DelimToken::Bracket, _) => match prev { - TokenTree::Token(token) => match token.kind { - token::Pound => false, - _ => true, - }, - _ => true, - }, + TokenTree::Token(token) => token.kind != token::Comma, + TokenTree::Delimited(_, DelimToken::Paren, _) => { + !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. })) + } + TokenTree::Delimited(_, DelimToken::Bracket, _) => { + !matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. })) + } TokenTree::Delimited(..) => true, } } @@ -650,13 +639,11 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere fn break_offset_if_not_bol(&mut self, n: usize, off: isize) { if !self.is_beginning_of_line() { self.break_offset(n, off) - } else { - if off != 0 && self.last_token().is_hardbreak_tok() { - // We do something pretty sketchy here: tuck the nonzero - // offset-adjustment we were going to deposit along with the - // break into the previous hardbreak. - self.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); - } + } else if off != 0 && self.last_token().is_hardbreak_tok() { + // We do something pretty sketchy here: tuck the nonzero + // offset-adjustment we were going to deposit along with the + // break into the previous hardbreak. + self.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); } } diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 9c309345000..48238c8bbf5 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -901,38 +901,36 @@ pub fn find_repr_attrs(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> { ) .emit(); } - } else { - if let Some(meta_item) = item.meta_item() { - if meta_item.has_name(sym::align) { - if let MetaItemKind::NameValue(ref value) = meta_item.kind { - recognised = true; - let mut err = struct_span_err!( - diagnostic, - item.span(), - E0693, - "incorrect `repr(align)` attribute format" - ); - match value.kind { - ast::LitKind::Int(int, ast::LitIntType::Unsuffixed) => { - err.span_suggestion( - item.span(), - "use parentheses instead", - format!("align({})", int), - Applicability::MachineApplicable, - ); - } - ast::LitKind::Str(s, _) => { - err.span_suggestion( - item.span(), - "use parentheses instead", - format!("align({})", s), - Applicability::MachineApplicable, - ); - } - _ => {} + } else if let Some(meta_item) = item.meta_item() { + if meta_item.has_name(sym::align) { + if let MetaItemKind::NameValue(ref value) = meta_item.kind { + recognised = true; + let mut err = struct_span_err!( + diagnostic, + item.span(), + E0693, + "incorrect `repr(align)` attribute format" + ); + match value.kind { + ast::LitKind::Int(int, ast::LitIntType::Unsuffixed) => { + err.span_suggestion( + item.span(), + "use parentheses instead", + format!("align({})", int), + Applicability::MachineApplicable, + ); + } + ast::LitKind::Str(s, _) => { + err.span_suggestion( + item.span(), + "use parentheses instead", + format!("align({})", s), + Applicability::MachineApplicable, + ); } - err.emit(); + _ => {} } + err.emit(); } } } @@ -1013,13 +1011,28 @@ pub fn allow_internal_unstable<'a>( sess: &'a Session, attrs: &'a [Attribute], ) -> Option<impl Iterator<Item = Symbol> + 'a> { - let attrs = sess.filter_by_name(attrs, sym::allow_internal_unstable); + allow_unstable(sess, attrs, sym::allow_internal_unstable) +} + +pub fn rustc_allow_const_fn_unstable<'a>( + sess: &'a Session, + attrs: &'a [Attribute], +) -> Option<impl Iterator<Item = Symbol> + 'a> { + allow_unstable(sess, attrs, sym::rustc_allow_const_fn_unstable) +} + +fn allow_unstable<'a>( + sess: &'a Session, + attrs: &'a [Attribute], + symbol: Symbol, +) -> Option<impl Iterator<Item = Symbol> + 'a> { + let attrs = sess.filter_by_name(attrs, symbol); let list = attrs .filter_map(move |attr| { attr.meta_item_list().or_else(|| { sess.diagnostic().span_err( attr.span, - "`allow_internal_unstable` expects a list of feature names", + &format!("`{}` expects a list of feature names", symbol.to_ident_string()), ); None }) @@ -1029,8 +1042,10 @@ pub fn allow_internal_unstable<'a>( Some(list.into_iter().filter_map(move |it| { let name = it.ident().map(|ident| ident.name); if name.is_none() { - sess.diagnostic() - .span_err(it.span(), "`allow_internal_unstable` expects feature names"); + sess.diagnostic().span_err( + it.span(), + &format!("`{}` expects feature names", symbol.to_ident_string()), + ); } name })) diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs index 5ed8b69d92a..747e48ece70 100644 --- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs +++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs @@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - ); let start_span = parser.token.span; - let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() { + let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item(false) { Ok(ai) => ai, Err(mut err) => { err.emit(); diff --git a/compiler/rustc_codegen_cranelift/.gitattributes b/compiler/rustc_codegen_cranelift/.gitattributes new file mode 100644 index 00000000000..6313b56c578 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml new file mode 100644 index 00000000000..841e1a0870e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -0,0 +1,54 @@ +name: CI + +on: + - push + - pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v2 + + - name: Cache cargo installed crates + uses: actions/cache@v2 + with: + path: ~/.cargo/bin + key: ${{ runner.os }}-cargo-installed-crates + + - name: Cache cargo registry and index + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache cargo target dir + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Prepare dependencies + run: | + git config --global user.email "user@example.com" + git config --global user.name "User" + ./prepare.sh + + - name: Test + run: | + # Enable backtraces for easier debugging + export RUST_BACKTRACE=1 + + # Reduce amount of benchmark runs as they are slow + export COMPILE_RUNS=2 + export RUN_RUNS=2 + + ./test.sh --release diff --git a/compiler/rustc_codegen_cranelift/.gitignore b/compiler/rustc_codegen_cranelift/.gitignore new file mode 100644 index 00000000000..0da9927b479 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.gitignore @@ -0,0 +1,14 @@ +target +**/*.rs.bk +*.rlib +*.o +perf.data +perf.data.old +*.events +*.string* +/build_sysroot/sysroot +/build_sysroot/sysroot_src +/rust +/rand +/regex +/simple-raytracer diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json new file mode 100644 index 00000000000..04ab5085c19 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json @@ -0,0 +1,53 @@ +{ + // source for rustc_* is not included in the rust-src component; disable the errors about this + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"], + "rust-analyzer.cargo.loadOutDirsFromCheck": true, + "rust-analyzer.linkedProjects": [ + "./Cargo.toml", + //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml", + { + "roots": [ + "./example/mini_core.rs", + "./example/mini_core_hello_world.rs", + "./example/mod_bench.rs" + ], + "crates": [ + { + "root_module": "./example/mini_core.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + { + "root_module": "./example/mini_core_hello_world.rs", + "edition": "2018", + "deps": [{ "crate": 0, "name": "mini_core" }], + "cfg": [], + }, + { + "root_module": "./example/mod_bench.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + ] + }, + { + "roots": ["./scripts/filter_profile.rs"], + "crates": [ + { + "root_module": "./scripts/filter_profile.rs", + "edition": "2018", + "deps": [{ "crate": 1, "name": "std" }], + "cfg": [], + }, + { + "root_module": "./build_sysroot/sysroot_src/library/std/src/lib.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + ] + } + ] +} diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock new file mode 100644 index 00000000000..6cfbed0a5e4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -0,0 +1,425 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1fd36ffbb1fb7c834eac128ea8d0e310c5aeb635548f9d58861e1308d46e71c" + +[[package]] +name = "ar" +version = "0.8.0" +source = "git+https://github.com/bjorn3/rust-ar.git?branch=do_not_remove_cg_clif_ranlib#de9ab0e56bf3a208381d342aa5b60f9ff2891648" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cranelift-bforest" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "byteorder", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-entity", + "gimli", + "log", + "regalloc", + "smallvec", + "target-lexicon", + "thiserror", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "cranelift-codegen-shared", + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" + +[[package]] +name = "cranelift-entity" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" + +[[package]] +name = "cranelift-frontend" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-module" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-entity", + "log", + "thiserror", +] + +[[package]] +name = "cranelift-native" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "cranelift-codegen", + "raw-cpuid", + "target-lexicon", +] + +[[package]] +name = "cranelift-object" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-module", + "log", + "object", + "target-lexicon", +] + +[[package]] +name = "cranelift-simplejit" +version = "0.67.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#4fd90dccabb266e983740e1f5daf8bde9266b286" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-module", + "cranelift-native", + "errno", + "libc", + "log", + "region", + "target-lexicon", + "winapi", +] + +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eab5ee3df98a279d9b316b1af6ac95422127b1290317e6d18c1743c99418b01" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" +dependencies = [ + "gcc", + "libc", +] + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +dependencies = [ + "indexmap", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "libc" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" + +[[package]] +name = "libloading" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3557c9384f7f757f6d139cd3a4c62ef4e850696c16bf27924a5538c8a09717a1" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" +dependencies = [ + "crc32fast", + "indexmap", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "raw-cpuid" +version = "7.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a349ca83373cfa5d6dbb66fd76e58b2cca08da71a5f6400de0a0a6a9bceeaf" +dependencies = [ + "bitflags", + "cc", + "rustc_version", +] + +[[package]] +name = "regalloc" +version = "0.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" +dependencies = [ + "log", + "rustc-hash", + "smallvec", +] + +[[package]] +name = "region" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +dependencies = [ + "bitflags", + "libc", + "mach", + "winapi", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_codegen_cranelift" +version = "0.1.0" +dependencies = [ + "ar", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-module", + "cranelift-object", + "cranelift-simplejit", + "gimli", + "indexmap", + "libloading", + "object", + "target-lexicon", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "smallvec" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" + +[[package]] +name = "syn" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "target-lexicon" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2635952a442a01fd4cb53d98858b5e4bb461b02c0d111f22f31772e3e7a8b2" + +[[package]] +name = "thiserror" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml new file mode 100644 index 00000000000..1c8e350d242 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -0,0 +1,76 @@ +[package] +name = "rustc_codegen_cranelift" +version = "0.1.0" +authors = ["bjorn3 <bjorn3@users.noreply.github.com>"] +edition = "2018" + +[lib] +crate-type = ["dylib"] + +[dependencies] +# These have to be in sync with each other +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } +cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } +cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +target-lexicon = "0.11.0" +gimli = { version = "0.22.0", default-features = false, features = ["write"]} +object = { version = "0.21.1", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] } + +ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } +indexmap = "1.0.2" +libloading = { version = "0.6.0", optional = true } + +# Uncomment to use local checkout of cranelift +#[patch."https://github.com/bytecodealliance/wasmtime/"] +#cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } +#cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } +#cranelift-module = { path = "../wasmtime/cranelift/module" } +#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" } +#cranelift-object = { path = "../wasmtime/cranelift/object" } + +#[patch.crates-io] +#gimli = { path = "../" } + +[features] +default = ["jit", "inline_asm"] +jit = ["cranelift-simplejit", "libloading"] +inline_asm = [] + +[profile.dev] +# By compiling dependencies with optimizations, performing tests gets much faster. +opt-level = 3 + +[profile.dev.package.rustc_codegen_cranelift] +# Disabling optimizations for cg_clif itself makes compilation after a change faster. +opt-level = 0 + +[profile.release.package.rustc_codegen_cranelift] +incremental = true + +# Disable optimizations and debuginfo of build scripts and some of the heavy build deps, as the +# execution time of build scripts is so fast that optimizing them slows down the total build time. +[profile.dev.build-override] +opt-level = 0 +debug = false + +[profile.release.build-override] +opt-level = 0 +debug = false + +[profile.dev.package.cranelift-codegen-meta] +opt-level = 0 +debug = false + +[profile.release.package.cranelift-codegen-meta] +opt-level = 0 +debug = false + +[profile.dev.package.syn] +opt-level = 0 +debug = false + +[profile.release.package.syn] +opt-level = 0 +debug = false diff --git a/compiler/rustc_codegen_cranelift/LICENSE-APACHE b/compiler/rustc_codegen_cranelift/LICENSE-APACHE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/compiler/rustc_codegen_cranelift/LICENSE-MIT b/compiler/rustc_codegen_cranelift/LICENSE-MIT new file mode 100644 index 00000000000..31aa79387f2 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md new file mode 100644 index 00000000000..680ff877656 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/Readme.md @@ -0,0 +1,88 @@ +# WIP Cranelift codegen backend for rust + +> âš âš âš Certain kinds of FFI don't work yet. âš âš âš + +The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift). This has the potential to improve compilation times in debug mode. If your project doesn't use any of the things listed under "Not yet supported", it should work fine. If not please open an issue. + +## Building + +```bash +$ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git +$ cd rustc_codegen_cranelift +$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking +$ ./test.sh --release +``` + +## Usage + +rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. + +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `test.sh`). + +### Cargo + +In the directory with your project (where you can do the usual `cargo build`), run: + +```bash +$ $cg_clif_dir/cargo.sh run +``` + +This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend. + +If you compiled cg_clif in debug mode (aka you didn't pass `--release` to `./test.sh`) you should set `CHANNEL="debug"`. + +### Rustc + +> You should prefer using the Cargo method. + +```bash +$ $cg_clif_dir/target/release/cg_clif my_crate.rs +``` + +### Jit mode + +In jit mode cg_clif will immediately execute your code without creating an executable file. + +> This requires all dependencies to be available as dynamic library. +> The jit mode will probably need cargo integration to make this possible. + +```bash +$ $cg_clif_dir/cargo.sh jit +``` + +or + +```bash +$ $cg_clif_dir/target/release/cg_clif --jit my_crate.rs +``` + +### Shell + +These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit. + +```bash +function jit_naked() { + echo "$@" | $cg_clif_dir/target/release/cg_clif - --jit +} + +function jit() { + jit_naked "fn main() { $@ }" +} + +function jit_calc() { + jit 'println!("0x{:x}", ' $@ ');'; +} +``` + +## Env vars + +[see env_vars.md](docs/env_vars.md) + +## Not yet supported + +* Good non-rust abi support ([several problems](https://github.com/bjorn3/rustc_codegen_cranelift/issues/10)) +* Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041) + * On Linux there is support for invoking an external assembler for `global_asm!` and `asm!`. + `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You + have to specify specific registers instead. +* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work) diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock new file mode 100644 index 00000000000..03ba5b53d2e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock @@ -0,0 +1,324 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "compiler_builtins", + "gimli", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "alloc" +version = "0.0.0" +dependencies = [ + "compiler_builtins", + "core", +] + +[[package]] +name = "alloc_system" +version = "0.0.0" +dependencies = [ + "compiler_builtins", + "core", + "libc", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "compiler_builtins" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0782e0a7da7598164153173e5a5d4d9b1da094473c98dce0ff91406112369" +dependencies = [ + "rustc-std-workspace-core", +] + +[[package]] +name = "core" +version = "0.0.0" + +[[package]] +name = "dlmalloc" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35055b1021724f4eb5262eb49130eebff23fc59fc5a14160e05faad8eeb36673" +dependencies = [ + "compiler_builtins", + "libc", + "rustc-std-workspace-core", +] + +[[package]] +name = "fortanix-sgx-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56c422ef86062869b2d57ae87270608dc5929969dd130a6e248979cf4fb6ca6" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "rustc-std-workspace-core", + "rustc-std-workspace-std", + "unicode-width", +] + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "compiler_builtins", + "libc", + "rustc-std-workspace-core", +] + +[[package]] +name = "libc" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" +dependencies = [ + "rustc-std-workspace-core", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + +[[package]] +name = "panic_abort" +version = "0.0.0" +dependencies = [ + "cfg-if", + "compiler_builtins", + "core", + "libc", +] + +[[package]] +name = "panic_unwind" +version = "0.0.0" +dependencies = [ + "alloc", + "cfg-if", + "compiler_builtins", + "core", + "libc", + "unwind", +] + +[[package]] +name = "proc_macro" +version = "0.0.0" +dependencies = [ + "std", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", +] + +[[package]] +name = "rustc-std-workspace-alloc" +version = "1.99.0" +dependencies = [ + "alloc", +] + +[[package]] +name = "rustc-std-workspace-core" +version = "1.99.0" +dependencies = [ + "core", +] + +[[package]] +name = "rustc-std-workspace-std" +version = "1.99.0" +dependencies = [ + "std", +] + +[[package]] +name = "std" +version = "0.0.0" +dependencies = [ + "addr2line", + "alloc", + "cfg-if", + "compiler_builtins", + "core", + "dlmalloc", + "fortanix-sgx-abi", + "hashbrown", + "hermit-abi", + "libc", + "miniz_oxide", + "object", + "panic_abort", + "panic_unwind", + "rustc-demangle", + "unwind", + "wasi", +] + +[[package]] +name = "sysroot" +version = "0.0.0" +dependencies = [ + "alloc", + "alloc_system", + "compiler_builtins", + "core", + "std", + "test", +] + +[[package]] +name = "term" +version = "0.0.0" +dependencies = [ + "core", + "std", +] + +[[package]] +name = "test" +version = "0.0.0" +dependencies = [ + "cfg-if", + "core", + "getopts", + "libc", + "panic_abort", + "panic_unwind", + "proc_macro", + "std", + "term", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-core", + "rustc-std-workspace-std", +] + +[[package]] +name = "unwind" +version = "0.0.0" +dependencies = [ + "cc", + "cfg-if", + "compiler_builtins", + "core", + "libc", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +dependencies = [ + "compiler_builtins", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml new file mode 100644 index 00000000000..e562dedb532 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml @@ -0,0 +1,26 @@ +[package] +authors = ["bjorn3 <bjorn3@users.noreply.github.com>"] +name = "sysroot" +version = "0.0.0" + +[dependencies] +core = { path = "./sysroot_src/library/core" } +compiler_builtins = "0.1" +alloc = { path = "./sysroot_src/library/alloc" } +std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } +test = { path = "./sysroot_src/library/test" } + +alloc_system = { path = "./alloc_system" } + +[patch.crates-io] +rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } +rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } +rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" } + +[profile.dev] +lto = "off" + +[profile.release] +debug = true +incremental = true +lto = "off" diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml b/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml new file mode 100644 index 00000000000..9fffca84300 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml @@ -0,0 +1,13 @@ +[package] +authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"] +name = "alloc_system" +version = "0.0.0" +[lib] +name = "alloc_system" +path = "lib.rs" +test = false +doc = false +[dependencies] +core = { path = "../sysroot_src/library/core" } +libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false } +compiler_builtins = "0.1" diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs b/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs new file mode 100644 index 00000000000..ca145e4f2a5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/lib.rs @@ -0,0 +1,342 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +#![no_std] +#![allow(unused_attributes)] +#![unstable(feature = "alloc_system", + reason = "this library is unlikely to be stabilized in its current \ + form or name", + issue = "32838")] +#![feature(allocator_api)] +#![feature(core_intrinsics)] +#![feature(nll)] +#![feature(staged_api)] +#![feature(rustc_attrs)] +#![feature(alloc_layout_extra)] +#![cfg_attr( + all(target_arch = "wasm32", not(target_os = "emscripten")), + feature(integer_atomics, stdsimd) +)] +#![cfg_attr(any(unix, target_os = "cloudabi", target_os = "redox"), feature(libc))] +// The minimum alignment guaranteed by the architecture. This value is used to +// add fast paths for low alignment values. +#[cfg(all(any(target_arch = "x86", + target_arch = "arm", + target_arch = "mips", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "asmjs", + target_arch = "wasm32")))] +#[allow(dead_code)] +const MIN_ALIGN: usize = 8; +#[cfg(all(any(target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "mips64", + target_arch = "s390x", + target_arch = "sparc64")))] +#[allow(dead_code)] +const MIN_ALIGN: usize = 16; + +/// The default memory allocator provided by the operating system. +/// +/// This is based on `malloc` on Unix platforms and `HeapAlloc` on Windows, +/// plus related functions. +/// +/// This type can be used in a `static` item +/// with the `#[global_allocator]` attribute +/// to force the global allocator to be the system’s one. +/// (The default is jemalloc for executables, on some platforms.) +/// +/// ```rust +/// use std::alloc::System; +/// +/// #[global_allocator] +/// static A: System = System; +/// +/// fn main() { +/// let a = Box::new(4); // Allocates from the system allocator. +/// println!("{}", a); +/// } +/// ``` +/// +/// It can also be used directly to allocate memory +/// independently of the standard library’s global allocator. +#[stable(feature = "alloc_system_type", since = "1.28.0")] +pub struct System; +#[cfg(any(windows, unix, target_os = "cloudabi", target_os = "redox"))] +mod realloc_fallback { + use core::alloc::{GlobalAlloc, Layout}; + use core::cmp; + use core::ptr; + impl super::System { + pub(crate) unsafe fn realloc_fallback(&self, ptr: *mut u8, old_layout: Layout, + new_size: usize) -> *mut u8 { + // Docs for GlobalAlloc::realloc require this to be valid: + let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); + let new_ptr = GlobalAlloc::alloc(self, new_layout); + if !new_ptr.is_null() { + let size = cmp::min(old_layout.size(), new_size); + ptr::copy_nonoverlapping(ptr, new_ptr, size); + GlobalAlloc::dealloc(self, ptr, old_layout); + } + new_ptr + } + } +} +#[cfg(any(unix, target_os = "cloudabi", target_os = "redox"))] +mod platform { + extern crate libc; + use core::ptr; + use MIN_ALIGN; + use System; + use core::alloc::{GlobalAlloc, Layout}; + #[stable(feature = "alloc_system_type", since = "1.28.0")] + unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { + libc::malloc(layout.size()) as *mut u8 + } else { + #[cfg(target_os = "macos")] + { + if layout.align() > (1 << 31) { + return ptr::null_mut() + } + } + aligned_malloc(&layout) + } + } + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { + libc::calloc(layout.size(), 1) as *mut u8 + } else { + let ptr = self.alloc(layout.clone()); + if !ptr.is_null() { + ptr::write_bytes(ptr, 0, layout.size()); + } + ptr + } + } + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + libc::free(ptr as *mut libc::c_void) + } + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if layout.align() <= MIN_ALIGN && layout.align() <= new_size { + libc::realloc(ptr as *mut libc::c_void, new_size) as *mut u8 + } else { + self.realloc_fallback(ptr, layout, new_size) + } + } + } + #[cfg(any(target_os = "android", + target_os = "hermit", + target_os = "redox", + target_os = "solaris"))] + #[inline] + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + // On android we currently target API level 9 which unfortunately + // doesn't have the `posix_memalign` API used below. Instead we use + // `memalign`, but this unfortunately has the property on some systems + // where the memory returned cannot be deallocated by `free`! + // + // Upon closer inspection, however, this appears to work just fine with + // Android, so for this platform we should be fine to call `memalign` + // (which is present in API level 9). Some helpful references could + // possibly be chromium using memalign [1], attempts at documenting that + // memalign + free is ok [2] [3], or the current source of chromium + // which still uses memalign on android [4]. + // + // [1]: https://codereview.chromium.org/10796020/ + // [2]: https://code.google.com/p/android/issues/detail?id=35391 + // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 + // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ + // /memory/aligned_memory.cc + libc::memalign(layout.align(), layout.size()) as *mut u8 + } + #[cfg(not(any(target_os = "android", + target_os = "hermit", + target_os = "redox", + target_os = "solaris")))] + #[inline] + unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { + let mut out = ptr::null_mut(); + let ret = libc::posix_memalign(&mut out, layout.align(), layout.size()); + if ret != 0 { + ptr::null_mut() + } else { + out as *mut u8 + } + } +} +#[cfg(windows)] +#[allow(nonstandard_style)] +mod platform { + use MIN_ALIGN; + use System; + use core::alloc::{GlobalAlloc, Layout}; + type LPVOID = *mut u8; + type HANDLE = LPVOID; + type SIZE_T = usize; + type DWORD = u32; + type BOOL = i32; + extern "system" { + fn GetProcessHeap() -> HANDLE; + fn HeapAlloc(hHeap: HANDLE, dwFlags: DWORD, dwBytes: SIZE_T) -> LPVOID; + fn HeapReAlloc(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID, dwBytes: SIZE_T) -> LPVOID; + fn HeapFree(hHeap: HANDLE, dwFlags: DWORD, lpMem: LPVOID) -> BOOL; + fn GetLastError() -> DWORD; + } + #[repr(C)] + struct Header(*mut u8); + const HEAP_ZERO_MEMORY: DWORD = 0x00000008; + unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { + &mut *(ptr as *mut Header).offset(-1) + } + unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { + let aligned = ptr.add(align - (ptr as usize & (align - 1))); + *get_header(aligned) = Header(ptr); + aligned + } + #[inline] + unsafe fn allocate_with_flags(layout: Layout, flags: DWORD) -> *mut u8 { + let ptr = if layout.align() <= MIN_ALIGN { + HeapAlloc(GetProcessHeap(), flags, layout.size()) + } else { + let size = layout.size() + layout.align(); + let ptr = HeapAlloc(GetProcessHeap(), flags, size); + if ptr.is_null() { + ptr + } else { + align_ptr(ptr, layout.align()) + } + }; + ptr as *mut u8 + } + #[stable(feature = "alloc_system_type", since = "1.28.0")] + unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + allocate_with_flags(layout, 0) + } + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + allocate_with_flags(layout, HEAP_ZERO_MEMORY) + } + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if layout.align() <= MIN_ALIGN { + let err = HeapFree(GetProcessHeap(), 0, ptr as LPVOID); + debug_assert!(err != 0, "Failed to free heap memory: {}", + GetLastError()); + } else { + let header = get_header(ptr); + let err = HeapFree(GetProcessHeap(), 0, header.0 as LPVOID); + debug_assert!(err != 0, "Failed to free heap memory: {}", + GetLastError()); + } + } + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if layout.align() <= MIN_ALIGN { + HeapReAlloc(GetProcessHeap(), 0, ptr as LPVOID, new_size) as *mut u8 + } else { + self.realloc_fallback(ptr, layout, new_size) + } + } + } +} +// This is an implementation of a global allocator on the wasm32 platform when +// emscripten is not in use. In that situation there's no actual runtime for us +// to lean on for allocation, so instead we provide our own! +// +// The wasm32 instruction set has two instructions for getting the current +// amount of memory and growing the amount of memory. These instructions are the +// foundation on which we're able to build an allocator, so we do so! Note that +// the instructions are also pretty "global" and this is the "global" allocator +// after all! +// +// The current allocator here is the `dlmalloc` crate which we've got included +// in the rust-lang/rust repository as a submodule. The crate is a port of +// dlmalloc.c from C to Rust and is basically just so we can have "pure Rust" +// for now which is currently technically required (can't link with C yet). +// +// The crate itself provides a global allocator which on wasm has no +// synchronization as there are no threads! +#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))] +mod platform { + extern crate dlmalloc; + use core::alloc::{GlobalAlloc, Layout}; + use System; + static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT; + #[stable(feature = "alloc_system_type", since = "1.28.0")] + unsafe impl GlobalAlloc for System { + #[inline] + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let _lock = lock::lock(); + DLMALLOC.malloc(layout.size(), layout.align()) + } + #[inline] + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let _lock = lock::lock(); + DLMALLOC.calloc(layout.size(), layout.align()) + } + #[inline] + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let _lock = lock::lock(); + DLMALLOC.free(ptr, layout.size(), layout.align()) + } + #[inline] + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let _lock = lock::lock(); + DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size) + } + } + #[cfg(target_feature = "atomics")] + mod lock { + use core::arch::wasm32; + use core::sync::atomic::{AtomicI32, Ordering::SeqCst}; + static LOCKED: AtomicI32 = AtomicI32::new(0); + pub struct DropLock; + pub fn lock() -> DropLock { + loop { + if LOCKED.swap(1, SeqCst) == 0 { + return DropLock + } + unsafe { + let r = wasm32::atomic::wait_i32( + &LOCKED as *const AtomicI32 as *mut i32, + 1, // expected value + -1, // timeout + ); + debug_assert!(r == 0 || r == 1); + } + } + } + impl Drop for DropLock { + fn drop(&mut self) { + let r = LOCKED.swap(0, SeqCst); + debug_assert_eq!(r, 1); + unsafe { + wasm32::atomic::wake( + &LOCKED as *const AtomicI32 as *mut i32, + 1, // only one thread + ); + } + } + } + } + #[cfg(not(target_feature = "atomics"))] + mod lock { + #[inline] + pub fn lock() {} // no atomics, no threads, that's easy! + } +} diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh b/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh new file mode 100755 index 00000000000..04c82ca2a51 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Requires the CHANNEL env var to be set to `debug` or `release.` + +set -e +cd $(dirname "$0") + +pushd ../ >/dev/null +source ./scripts/config.sh +popd >/dev/null + +# Cleanup for previous run +# v Clean target dir except for build scripts and incremental cache +rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true +rm -r sysroot/ 2>/dev/null || true + +# Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that +# build scripts are still compiled using cg_llvm. +export RUSTC=$(pwd)/../"target/"$CHANNEL"/cg_clif_build_sysroot" +export RUSTFLAGS=$RUSTFLAGS" --clif" + +# Build libs +export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" +if [[ "$1" == "--release" ]]; then + sysroot_channel='release' + # FIXME Enable incremental again once rust-lang/rust#74946 is fixed + # FIXME Enable -Zmir-opt-level=2 again once it doesn't ice anymore + CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS" cargo build --target $TARGET_TRIPLE --release +else + sysroot_channel='debug' + cargo build --target $TARGET_TRIPLE +fi + +# Copy files to sysroot +mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ +cp -r target/$TARGET_TRIPLE/$sysroot_channel/deps/* sysroot/lib/rustlib/$TARGET_TRIPLE/lib/ diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh new file mode 100755 index 00000000000..14aa77478f5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e +cd $(dirname "$0") + +SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/" +DST_DIR="sysroot_src" + +if [ ! -e $SRC_DIR ]; then + echo "Please install rust-src component" + exit 1 +fi + +rm -rf $DST_DIR +mkdir -p $DST_DIR/library +cp -r $SRC_DIR/library $DST_DIR/ + +pushd $DST_DIR +echo "[GIT] init" +git init +echo "[GIT] add" +git add . +echo "[GIT] commit" +git commit -m "Initial commit" -q +for file in $(ls ../../patches/ | grep -v patcha); do +echo "[GIT] apply" $file +git apply ../../patches/$file +git add -A +git commit --no-gpg-sign -m "Patch $file" +done +popd + +echo "Successfully prepared libcore for building" diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/src/lib.rs b/compiler/rustc_codegen_cranelift/build_sysroot/src/lib.rs new file mode 100644 index 00000000000..0c9ac1ac8e4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/build_sysroot/src/lib.rs @@ -0,0 +1 @@ +#![no_std] diff --git a/compiler/rustc_codegen_cranelift/cargo.sh b/compiler/rustc_codegen_cranelift/cargo.sh new file mode 100755 index 00000000000..cebc3e67363 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/cargo.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ -z $CHANNEL ]; then +export CHANNEL='release' +fi + +pushd $(dirname "$0") >/dev/null +source scripts/config.sh + +# read nightly compiler from rust-toolchain file +TOOLCHAIN=$(cat rust-toolchain) + +popd >/dev/null + +cmd=$1 +shift + +if [[ "$cmd" = "jit" ]]; then +cargo +${TOOLCHAIN} rustc $@ -- --jit +else +cargo +${TOOLCHAIN} $cmd $@ +fi diff --git a/compiler/rustc_codegen_cranelift/clean_all.sh b/compiler/rustc_codegen_cranelift/clean_all.sh new file mode 100755 index 00000000000..3003a0ea2d1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/clean_all.sh @@ -0,0 +1,5 @@ +#!/bin/bash --verbose +set -e + +rm -rf target/ build_sysroot/{sysroot/,sysroot_src/,target/} perf.data{,.old} +rm -rf rand/ regex/ simple-raytracer/ diff --git a/compiler/rustc_codegen_cranelift/crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch b/compiler/rustc_codegen_cranelift/crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch new file mode 100644 index 00000000000..01dc0fcc537 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch @@ -0,0 +1,23 @@ +From 9c5663e36391fa20becf84f3af2e82afa5bb720b Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Sat, 15 Aug 2020 19:56:03 +0200 +Subject: [PATCH] [rand] Enable c2-chacha simd feature + +--- + rand_chacha/Cargo.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/rand_chacha/Cargo.toml b/rand_chacha/Cargo.toml +index 9190b7f..872cca2 100644 +--- a/rand_chacha/Cargo.toml ++++ b/rand_chacha/Cargo.toml +@@ -24,5 +24,5 @@ ppv-lite86 = { version = "0.2.8", default-features = false } + + [features] + default = ["std"] +-std = ["ppv-lite86/std"] ++std = ["ppv-lite86/std", "ppv-lite86/simd"] + simd = [] # deprecated +-- +2.20.1 + diff --git a/compiler/rustc_codegen_cranelift/crate_patches/0002-rand-Disable-failing-test.patch b/compiler/rustc_codegen_cranelift/crate_patches/0002-rand-Disable-failing-test.patch new file mode 100644 index 00000000000..19fd20d7269 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/crate_patches/0002-rand-Disable-failing-test.patch @@ -0,0 +1,33 @@ +From a8fb97120d71252538b6b026695df40d02696bdb Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Sat, 15 Aug 2020 20:04:38 +0200 +Subject: [PATCH] [rand] Disable failing test + +--- + src/distributions/uniform.rs | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/src/distributions/uniform.rs b/src/distributions/uniform.rs +index 480b859..c80bb6f 100644 +--- a/src/distributions/uniform.rs ++++ b/src/distributions/uniform.rs +@@ -1085,7 +1085,7 @@ mod tests { + _ => panic!("`UniformDurationMode` was not serialized/deserialized correctly") + } + } +- ++ + #[test] + #[cfg(feature = "serde1")] + fn test_uniform_serialization() { +@@ -1314,6 +1314,7 @@ mod tests { + not(target_arch = "wasm32"), + not(target_arch = "asmjs") + ))] ++ #[ignore] // FIXME + fn test_float_assertions() { + use super::SampleUniform; + use std::panic::catch_unwind; +-- +2.20.1 + diff --git a/compiler/rustc_codegen_cranelift/docs/dwarf.md b/compiler/rustc_codegen_cranelift/docs/dwarf.md new file mode 100644 index 00000000000..502b1b03623 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/docs/dwarf.md @@ -0,0 +1,153 @@ +# Line number information + +Line number information maps between machine code instructions and the source level location. + +## Encoding + +The line number information is stored in the `.debug_line` section for ELF and `__debug_line` +section of the `__DWARF` segment for Mach-O object files. The line number information contains a +header followed by the line program. The line program is a program for a virtual machine with +instructions like set line number for the current machine code instruction and advance the current +machine code instruction. + +## Tips + +You need to set either `DW_AT_low_pc` and `DW_AT_high_pc` **or** `DW_AT_ranges` of a +`DW_TAG_compilation_unit` to the range of addresses in the compilation unit. After that you need +to set `DW_AT_stmt_list` to the `.debug_line` section offset of the line program. Otherwise a +debugger won't find the line number information. On macOS the debuginfo relocations **must** be +section relative and not symbol relative. +See [#303 (comment)](https://github.com/bjorn3/rustc_codegen_cranelift/issues/303#issuecomment-457825535) +for more information. + +# Function debuginfo + +## Tips + +`DW_TAG_subprogram` requires `DW_AT_name`, `DW_AT_low_pc` and `DW_AT_high_pc` **or** `DW_AT_ranges`. +Otherwise gdb will silently skip it. When `DW_AT_high_pc` is a length instead of an address, the +DWARF version must be at least 4. + +<details> +<summary>IRC log of #gdb on irc.freenode.org at 2020-04-23</summary> + +``` +(13:46:11) bjorn3: i am writing a backend for a compiler that uses DWARF for debuginfo. for some reason gdb seems to completely ignore all DW_TAG_subprogram, while lldb works fine. any idea what the problem could be? +(13:47:49) bjorn3: this is the output of llvm-dwarfdump: https://gist.github.com/bjorn3/8a34e333c80f13cb048381e94b4a3756 +(13:47:50) osa1: luispm: why is that problem not exists in 'commands'? (the target vs. host) +(13:52:16) luispm: osa1, commands is a bit more high level. It executes isolated commands. Breakpoint conditions need to be evaluated in the context of a valid expression. That expression may involve variables, symbols etc. +(13:52:36) luispm: osa1, Oh, i see your point now. Commands is only executed on the host. +(13:53:18) luispm: osa1, The commands are not tied to the execution context of the debugged program. The breakpoint conditions determine if execution must stop or continue etc. +(13:55:00) luispm: bjorn3, Likely something GDB thinks is wrong. Does enabling "set debug dwarf*" show anything? +(13:56:01) bjorn3: luispm: no +(13:56:12) bjorn3: for more context: https://github.com/bjorn3/rustc_codegen_cranelift/pull/978 +(13:58:16) osa1 verliet de ruimte (quit: Quit: osa1). +(13:58:28) bjorn3: luispm: wait, for b m<TAB> it shows nothing, but when stepping into a new function it does +(13:58:45) bjorn3: it still doesn't show anything for `info args` though +(13:58:50) bjorn3: No symbol table info available. +(14:00:50) luispm: bjorn3, Is that expected given the nature of the binary? +(14:01:17) bjorn3: b main<TAB> may show nothing as I only set DW_AT_linkage_name and not DW_AT_name +(14:01:24) bjorn3: info args should work though +(14:03:26) luispm: Sorry, I'm not sure what's up. There may be a genuine bug there. +(14:03:41) luispm: tromey (not currently in the channel, but maybe later today) may have more input. +(14:04:08) bjorn3: okay, thanks luispm! +(14:04:27) luispm: In the worst case, reporting a bug may prompt someone to look into that as well. +(14:04:48) luispm: Or send an e-mail to the gdb@sourceware.org mailing list. +(14:05:11) bjorn3: I don't know if it is a bug in gdb, or just me producing (slightly) wrong DWARF +(14:39:40) irker749: gdb: tom binutils-gdb.git:master * 740480b88af / gdb/ChangeLog gdb/darwin-nat.c gdb/inferior.c gdb/inferior.h: Remove iterate_over_inferiors +(15:22:45) irker749: gdb: tromey binutils-gdb.git:master * ecc6c6066b5 / gdb/ChangeLog gdb/dwarf2/read.c gdb/unittests/lookup_name_info-selftests.c: Fix Ada crash with .debug_names +(15:23:13) bjorn3: tromey: ping +(15:23:29) tromey: bjorn3: hey +(15:24:16) bjorn3: I am writing a backend for a compiler which uses DWARF for debuginfo. I unfortunately can't get gdb to show arguments. lldb works fine. +(15:25:13) bjorn3: it just says: No symbol table info available. +(15:25:21) bjorn3: any idea what it could be? +(15:25:34) bjorn3: dwarfdump output: https://gist.github.com/bjorn3/8a34e333c80f13cb048381e94b4a3756 +(15:26:48) bjorn3: more context: https://github.com/bjorn3/rustc_codegen_cranelift/pull/978 +(15:28:05) tromey: offhand I don't know, but if you can send me an executable I can look +(15:28:17) bjorn3: how should I send it? +(15:29:26) tromey: good question +(15:29:41) tromey: you could try emailing it to tromey at adacore.com +(15:29:47) tromey: dunno if that will work or not +(15:30:26) bjorn3: i will try +(15:37:27) bjorn3: tromey: i sent an email with the subject "gdb args not showing" +(15:38:29) tromey: will check now +(15:38:40) bjorn3: thanks! +(15:42:51) irker749: gdb: tdevries binutils-gdb.git:master * de82891ce5b / gdb/ChangeLog gdb/block.c gdb/block.h gdb/symtab.c gdb/testsuite/ChangeLog gdb/testsuite/gdb.base/decl-before-def-decl.c gdb/testsuite/gdb.base/decl-before-def-def.c gdb/testsuite/gdb.base/decl-before-def.exp: [gdb/symtab] Prefer def over decl (inter-CU case) +(15:42:52) irker749: gdb: tdevries binutils-gdb.git:master * 70bc38f5138 / gdb/ChangeLog gdb/symtab.c gdb/testsuite/ChangeLog gdb/testsuite/gdb.base/decl-before-def.exp: [gdb/symtab] Prefer def over decl (inter-CU case, with context) +(15:43:36) tromey: bjorn3: sorry, got distracted. I have the file now +(15:45:35) tromey: my first thing when investigating was to enable complaints +(15:45:37) tromey: so I did +(15:45:40) tromey: set complaints 1000 +(15:45:42) tromey: then +(15:45:51) tromey: file -readnow mini_core_hello_world +(15:46:00) tromey: gdb printed just one style of complaint +(15:46:07) tromey: During symbol reading: missing name for subprogram DIE at 0x3f7 +(15:46:18) tromey: (which is really pretty good, most compilers manage to generate a bunch) +(15:46:29) tromey: and then the gdb DWARF reader says +(15:46:34) tromey: /* Ignore functions with missing or empty names. These are actually +(15:46:34) tromey: illegal according to the DWARF standard. */ +(15:46:34) tromey: if (name == NULL) +(15:46:34) tromey: { +(15:46:37) tromey: complaint (_("missing name for subprogram DIE at %s"), +(15:46:40) tromey: sect_offset_str (die->sect_off)); +(15:46:47) tromey: I wonder if that comment is correct though +(15:47:34) tromey: I guess pedantically maybe it is, DWARF 5 3.3.1 says +(15:47:43) tromey: The subroutine or entry point entry has a DW_AT_name attribute whose value is +(15:47:43) tromey: a null-terminated string containing the subroutine or entry point name. +(15:48:14) bjorn3: i tried set complaints, but it returned complaints for system files. i didn't know about file -readnow. +(15:48:21) tromey: cool +(15:48:26) bjorn3: i will try adding DW_AT_name +(15:48:45) tromey: without readnow unfortunately you get less stuff, because for whatever reason gdb has 2 separate DWARF scanners +(15:49:02) tromey: sort of anyway +(15:49:43) tromey: this seems kind of pedantic of gdb, like if there's a linkage name but no DW_AT_name, then why bail? +(15:50:01) tromey: also what about anonymous functions +(15:50:17) tromey: but anyway this explains the current situation and if you don't mind adding DW_AT_name, then that's probably simplest +(15:51:47) bjorn3: i added DW_AT_name. +(15:51:54) bjorn3: now it says cannot get low and high bounds for subprogram DIE at ... +(15:52:01) tromey: ugh +(15:52:10) bjorn3: i will add DW_AT_low_pc and DW_AT_high_pc +(15:52:15) tromey: /* Ignore functions with missing or invalid low and high pc attributes. */ +(15:52:37) tromey: you can also use DW_AT_ranges +(15:52:55) tromey: if you'd prefer +(15:53:08) bjorn3: already using DW_AT_ranges for DW_TAG_compilation_unit +(15:53:19) bjorn3: for individual functions, there are no gaps +(15:57:07) bjorn3: still the same error with DW_AT_low_pc and DW_AT_high_pc +(15:57:24) bjorn3: tromey: ^ +(15:58:08) tromey: hmmm +(15:58:30) bjorn3: should i send the new executable? +(15:58:31) tromey: send me another executable & I will debug +(15:58:33) tromey: yep +(15:59:23) bjorn3: sent as repy of the previous mail +(16:03:23) tromey: the low PC has DW_FORM_addr, but the high PC has DW_FORM_udata, which seems weird +(16:03:50) mjw: no +(16:03:54) tromey: no? +(16:04:00) mjw: I suggested that for the DWARF standard... +(16:04:05) mjw: sorry +(16:04:58) mjw: The idea was that instead of two relocations and two address wide fields, you have one address and a constant offset. +(16:05:05) tromey: ahh, I see the code now +(16:05:07) tromey: I forgot about this +(16:05:18) tromey: if (cu->header.version >= 4 && attr_high->form_is_constant ()) +(16:05:18) tromey: high += low; +(16:05:36) mjw: that second offset doesn't need a relocation and can often be packed in something small, like an uleb128 +(16:05:51) mjw: using udata might not be ideal though, but is allowed +(16:05:51) tromey: bjorn3: the problem is that this CU claims to be DWARF 3 but is using a DWARF 4 feature +(16:05:58) mjw: aha +(16:05:59) bjorn3: which one? +(16:06:03) ryoshu: hi +(16:06:08) tromey: high_pc (udata) 107 (+0x00000000000011b0 <_ZN21mini_core_hello_world5start17hec55b7ca64fc434eE>) +(16:06:08) tromey: +(16:06:12) ryoshu: just soft ping, I have a queue of patches :) +(16:06:22) tromey: using this as a length requires DWARF 4 +(16:06:36) tromey: for gdb at least it's fine to always emit DWARF 4 +(16:06:44) bjorn3: trying dwarf 4 now +(16:06:48) tromey: I think there are some DWARF 5 features still in the works but DWARF 4 should be solid AFAIK +(16:07:03) tromey: fini +(16:07:08) tromey: lol wrong window +(16:07:56) mjw: Maybe you can accept it for DWARF < 4. But if I remember correctly it might be that people might have been using udata as if it was an address... +(16:08:13) tromey: yeah, I vaguely recall this as well, though I'd expect there to be a comment +(16:08:21) mjw: Cannot really remember why it needed version >= 4. Maybe there was no good reason? +(16:08:32) bjorn3: tromey: it works!!!! thanks for all the help! +(16:08:41) tromey: my pleasure bjorn3 +``` + +</details> diff --git a/compiler/rustc_codegen_cranelift/docs/env_vars.md b/compiler/rustc_codegen_cranelift/docs/env_vars.md new file mode 100644 index 00000000000..07b75622a58 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/docs/env_vars.md @@ -0,0 +1,15 @@ +# List of env vars recognized by cg_clif + +<dl> + <dt>CG_CLIF_JIT_ARGS</dt> + <dd>When JIT mode is enable pass these arguments to the program.</dd> + <dt>CG_CLIF_INCR_CACHE_DISABLED</dt> + <dd>Don't cache object files in the incremental cache. Useful during development of cg_clif + to make it possible to use incremental mode for all analyses performed by rustc without caching + object files when their content should have been changed by a change to cg_clif.</dd> + <dt>CG_CLIF_DISPLAY_CG_TIME</dt> + <dd>If "1", display the time it took to perform codegen for a crate</dd> + <dt>CG_CLIF_FUNCTION_SECTIONS</dt> + <dd>Use a single section for each function. This will often reduce the executable size at the + cost of making linking significantly slower.</dd> +</dl> diff --git a/compiler/rustc_codegen_cranelift/example/alloc_example.rs b/compiler/rustc_codegen_cranelift/example/alloc_example.rs new file mode 100644 index 00000000000..dc2ad4c676e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/alloc_example.rs @@ -0,0 +1,37 @@ +#![feature(start, box_syntax, alloc_system, core_intrinsics, alloc_prelude, alloc_error_handler)] +#![no_std] + +extern crate alloc; +extern crate alloc_system; + +use alloc::prelude::v1::*; + +use alloc_system::System; + +#[global_allocator] +static ALLOC: System = System; + +#[link(name = "c")] +extern "C" { + fn puts(s: *const u8) -> i32; +} + +#[panic_handler] +fn panic_handler(_: &core::panic::PanicInfo) -> ! { + core::intrinsics::abort(); +} + +#[alloc_error_handler] +fn alloc_error_handler(_: alloc::alloc::Layout) -> ! { + core::intrinsics::abort(); +} + +#[start] +fn main(_argc: isize, _argv: *const *const u8) -> isize { + let world: Box<&str> = box "Hello World!\0"; + unsafe { + puts(*world as *const str as *const u8); + } + + 0 +} diff --git a/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs new file mode 100644 index 00000000000..0b0039a1370 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs @@ -0,0 +1,82 @@ +// Adapted from rustc run-pass test suite + +#![feature(no_core, arbitrary_self_types, box_syntax)] +#![feature(rustc_attrs)] + +#![feature(start, lang_items)] +#![no_core] + +extern crate mini_core; + +use mini_core::*; + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + } +} + +struct Ptr<T: ?Sized>(Box<T>); + +impl<T: ?Sized> Deref for Ptr<T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.0 + } +} + +impl<T: Unsize<U> + ?Sized, U: ?Sized> CoerceUnsized<Ptr<U>> for Ptr<T> {} +impl<T: Unsize<U> + ?Sized, U: ?Sized> DispatchFromDyn<Ptr<U>> for Ptr<T> {} + +struct Wrapper<T: ?Sized>(T); + +impl<T: ?Sized> Deref for Wrapper<T> { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl<T: CoerceUnsized<U>, U> CoerceUnsized<Wrapper<U>> for Wrapper<T> {} +impl<T: DispatchFromDyn<U>, U> DispatchFromDyn<Wrapper<U>> for Wrapper<T> {} + + +trait Trait { + // This method isn't object-safe yet. Unsized by-value `self` is object-safe (but not callable + // without unsized_locals), but wrappers arond `Self` currently are not. + // FIXME (mikeyhew) uncomment this when unsized rvalues object-safety is implemented + // fn wrapper(self: Wrapper<Self>) -> i32; + fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32; + fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32; + fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32; +} + +impl Trait for i32 { + fn ptr_wrapper(self: Ptr<Wrapper<Self>>) -> i32 { + **self + } + fn wrapper_ptr(self: Wrapper<Ptr<Self>>) -> i32 { + **self + } + fn wrapper_ptr_wrapper(self: Wrapper<Ptr<Wrapper<Self>>>) -> i32 { + ***self + } +} + +#[start] +fn main(_: isize, _: *const *const u8) -> isize { + let pw = Ptr(box Wrapper(5)) as Ptr<Wrapper<dyn Trait>>; + assert_eq!(pw.ptr_wrapper(), 5); + + let wp = Wrapper(Ptr(box 6)) as Wrapper<Ptr<dyn Trait>>; + assert_eq!(wp.wrapper_ptr(), 6); + + let wpw = Wrapper(Ptr(box Wrapper(7))) as Wrapper<Ptr<Wrapper<dyn Trait>>>; + assert_eq!(wpw.wrapper_ptr_wrapper(), 7); + + 0 +} diff --git a/compiler/rustc_codegen_cranelift/example/dst-field-align.rs b/compiler/rustc_codegen_cranelift/example/dst-field-align.rs new file mode 100644 index 00000000000..6c338e99912 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/dst-field-align.rs @@ -0,0 +1,67 @@ +// run-pass +#![allow(dead_code)] +struct Foo<T: ?Sized> { + a: u16, + b: T +} + +trait Bar { + fn get(&self) -> usize; +} + +impl Bar for usize { + fn get(&self) -> usize { *self } +} + +struct Baz<T: ?Sized> { + a: T +} + +struct HasDrop<T: ?Sized> { + ptr: Box<usize>, + data: T +} + +fn main() { + // Test that zero-offset works properly + let b : Baz<usize> = Baz { a: 7 }; + assert_eq!(b.a.get(), 7); + let b : &Baz<dyn Bar> = &b; + assert_eq!(b.a.get(), 7); + + // Test that the field is aligned properly + let f : Foo<usize> = Foo { a: 0, b: 11 }; + assert_eq!(f.b.get(), 11); + let ptr1 : *const u8 = &f.b as *const _ as *const u8; + + let f : &Foo<dyn Bar> = &f; + let ptr2 : *const u8 = &f.b as *const _ as *const u8; + assert_eq!(f.b.get(), 11); + + // The pointers should be the same + assert_eq!(ptr1, ptr2); + + // Test that nested DSTs work properly + let f : Foo<Foo<usize>> = Foo { a: 0, b: Foo { a: 1, b: 17 }}; + assert_eq!(f.b.b.get(), 17); + let f : &Foo<Foo<dyn Bar>> = &f; + assert_eq!(f.b.b.get(), 17); + + // Test that get the pointer via destructuring works + + let f : Foo<usize> = Foo { a: 0, b: 11 }; + let f : &Foo<dyn Bar> = &f; + let &Foo { a: _, b: ref bar } = f; + assert_eq!(bar.get(), 11); + + // Make sure that drop flags don't screw things up + + let d : HasDrop<Baz<[i32; 4]>> = HasDrop { + ptr: Box::new(0), + data: Baz { a: [1,2,3,4] } + }; + assert_eq!([1,2,3,4], d.data.a); + + let d : &HasDrop<Baz<[i32]>> = &d; + assert_eq!(&[1,2,3,4], &d.data.a); +} diff --git a/compiler/rustc_codegen_cranelift/example/example.rs b/compiler/rustc_codegen_cranelift/example/example.rs new file mode 100644 index 00000000000..d5c122bf681 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/example.rs @@ -0,0 +1,208 @@ +#![feature(no_core, unboxed_closures)] +#![no_core] +#![allow(dead_code)] + +extern crate mini_core; + +use mini_core::*; + +pub fn abc(a: u8) -> u8 { + a * 2 +} + +pub fn bcd(b: bool, a: u8) -> u8 { + if b { + a * 2 + } else { + a * 3 + } +} + +pub fn call() { + abc(42); +} + +pub fn indirect_call() { + let f: fn() = call; + f(); +} + +pub enum BoolOption { + Some(bool), + None, +} + +pub fn option_unwrap_or(o: BoolOption, d: bool) -> bool { + match o { + BoolOption::Some(b) => b, + BoolOption::None => d, + } +} + +pub fn ret_42() -> u8 { + 42 +} + +pub fn return_str() -> &'static str { + "hello world" +} + +pub fn promoted_val() -> &'static u8 { + &(1 * 2) +} + +pub fn cast_ref_to_raw_ptr(abc: &u8) -> *const u8 { + abc as *const u8 +} + +pub fn cmp_raw_ptr(a: *const u8, b: *const u8) -> bool { + a == b +} + +pub fn int_cast(a: u16, b: i16) -> (u8, u16, u32, usize, i8, i16, i32, isize, u8, u32) { + ( + a as u8, a as u16, a as u32, a as usize, a as i8, a as i16, a as i32, a as isize, b as u8, + b as u32, + ) +} + +pub fn char_cast(c: char) -> u8 { + c as u8 +} + +pub struct DebugTuple(()); + +pub fn debug_tuple() -> DebugTuple { + DebugTuple(()) +} + +pub fn size_of<T>() -> usize { + intrinsics::size_of::<T>() +} + +pub fn use_size_of() -> usize { + size_of::<u64>() +} + +pub unsafe fn use_copy_intrinsic(src: *const u8, dst: *mut u8) { + intrinsics::copy::<u8>(src, dst, 1); +} + +pub unsafe fn use_copy_intrinsic_ref(src: *const u8, dst: *mut u8) { + let copy2 = &intrinsics::copy::<u8>; + copy2(src, dst, 1); +} + +pub const ABC: u8 = 6 * 7; + +pub fn use_const() -> u8 { + ABC +} + +pub fn call_closure_3arg() { + (|_, _, _| {})(0u8, 42u16, 0u8) +} + +pub fn call_closure_2arg() { + (|_, _| {})(0u8, 42u16) +} + +pub struct IsNotEmpty; + +impl<'a, 'b> FnOnce<(&'a &'b [u16],)> for IsNotEmpty { + type Output = (u8, u8); + + #[inline] + extern "rust-call" fn call_once(mut self, arg: (&'a &'b [u16],)) -> (u8, u8) { + self.call_mut(arg) + } +} + +impl<'a, 'b> FnMut<(&'a &'b [u16],)> for IsNotEmpty { + #[inline] + extern "rust-call" fn call_mut(&mut self, _arg: (&'a &'b [u16],)) -> (u8, u8) { + (0, 42) + } +} + +pub fn call_is_not_empty() { + IsNotEmpty.call_once((&(&[0u16] as &[_]),)); +} + +pub fn eq_char(a: char, b: char) -> bool { + a == b +} + +pub unsafe fn transmute(c: char) -> u32 { + intrinsics::transmute(c) +} + +pub unsafe fn deref_str_ptr(s: *const str) -> &'static str { + &*s +} + +pub fn use_array(arr: [u8; 3]) -> u8 { + arr[1] +} + +pub fn repeat_array() -> [u8; 3] { + [0; 3] +} + +pub fn array_as_slice(arr: &[u8; 3]) -> &[u8] { + arr +} + +pub unsafe fn use_ctlz_nonzero(a: u16) -> u16 { + intrinsics::ctlz_nonzero(a) +} + +pub fn ptr_as_usize(ptr: *const u8) -> usize { + ptr as usize +} + +pub fn float_cast(a: f32, b: f64) -> (f64, f32) { + (a as f64, b as f32) +} + +pub fn int_to_float(a: u8, b: i32) -> (f64, f32) { + (a as f64, b as f32) +} + +pub fn make_array() -> [u8; 3] { + [42, 0, 5] +} + +pub fn some_promoted_tuple() -> &'static (&'static str, &'static str) { + &("abc", "some") +} + +pub fn index_slice(s: &[u8]) -> u8 { + s[2] +} + +pub struct StrWrapper { + s: str, +} + +pub fn str_wrapper_get(w: &StrWrapper) -> &str { + &w.s +} + +pub fn i16_as_i8(a: i16) -> i8 { + a as i8 +} + +pub struct Unsized(u8, str); + +pub fn get_sized_field_ref_from_unsized_type(u: &Unsized) -> &u8 { + &u.0 +} + +pub fn get_unsized_field_ref_from_unsized_type(u: &Unsized) -> &str { + &u.1 +} + +pub fn reuse_byref_argument_storage(a: (u8, u16, u32)) -> u8 { + a.0 +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs new file mode 100644 index 00000000000..a972beedaa3 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -0,0 +1,603 @@ +#![feature( + no_core, lang_items, intrinsics, unboxed_closures, type_ascription, extern_types, + untagged_unions, decl_macro, rustc_attrs, transparent_unions, optin_builtin_traits, + thread_local, +)] +#![no_core] +#![allow(dead_code)] + +#[lang = "sized"] +pub trait Sized {} + +#[lang = "unsize"] +pub trait Unsize<T: ?Sized> {} + +#[lang = "coerce_unsized"] +pub trait CoerceUnsized<T> {} + +impl<'a, 'b: 'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a U> for &'b T {} +impl<'a, T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<&'a mut U> for &'a mut T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*const U> for *const T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<*mut U> for *mut T {} + +#[lang = "dispatch_from_dyn"] +pub trait DispatchFromDyn<T> {} + +// &T -> &U +impl<'a, T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<&'a U> for &'a T {} +// &mut T -> &mut U +impl<'a, T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<&'a mut U> for &'a mut T {} +// *const T -> *const U +impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*const U> for *const T {} +// *mut T -> *mut U +impl<T: ?Sized+Unsize<U>, U: ?Sized> DispatchFromDyn<*mut U> for *mut T {} +impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T> {} + +#[lang = "receiver"] +pub trait Receiver {} + +impl<T: ?Sized> Receiver for &T {} +impl<T: ?Sized> Receiver for &mut T {} +impl<T: ?Sized> Receiver for Box<T> {} + +#[lang = "copy"] +pub unsafe trait Copy {} + +unsafe impl Copy for bool {} +unsafe impl Copy for u8 {} +unsafe impl Copy for u16 {} +unsafe impl Copy for u32 {} +unsafe impl Copy for u64 {} +unsafe impl Copy for usize {} +unsafe impl Copy for i8 {} +unsafe impl Copy for i16 {} +unsafe impl Copy for i32 {} +unsafe impl Copy for isize {} +unsafe impl Copy for f32 {} +unsafe impl Copy for char {} +unsafe impl<'a, T: ?Sized> Copy for &'a T {} +unsafe impl<T: ?Sized> Copy for *const T {} +unsafe impl<T: ?Sized> Copy for *mut T {} +unsafe impl<T: Copy> Copy for Option<T> {} + +#[lang = "sync"] +pub unsafe trait Sync {} + +unsafe impl Sync for bool {} +unsafe impl Sync for u8 {} +unsafe impl Sync for u16 {} +unsafe impl Sync for u32 {} +unsafe impl Sync for u64 {} +unsafe impl Sync for usize {} +unsafe impl Sync for i8 {} +unsafe impl Sync for i16 {} +unsafe impl Sync for i32 {} +unsafe impl Sync for isize {} +unsafe impl Sync for char {} +unsafe impl<'a, T: ?Sized> Sync for &'a T {} +unsafe impl Sync for [u8; 16] {} + +#[lang = "freeze"] +unsafe auto trait Freeze {} + +unsafe impl<T: ?Sized> Freeze for PhantomData<T> {} +unsafe impl<T: ?Sized> Freeze for *const T {} +unsafe impl<T: ?Sized> Freeze for *mut T {} +unsafe impl<T: ?Sized> Freeze for &T {} +unsafe impl<T: ?Sized> Freeze for &mut T {} + +#[lang = "structural_peq"] +pub trait StructuralPartialEq {} + +#[lang = "structural_teq"] +pub trait StructuralEq {} + +#[lang = "not"] +pub trait Not { + type Output; + + fn not(self) -> Self::Output; +} + +impl Not for bool { + type Output = bool; + + fn not(self) -> bool { + !self + } +} + +#[lang = "mul"] +pub trait Mul<RHS = Self> { + type Output; + + #[must_use] + fn mul(self, rhs: RHS) -> Self::Output; +} + +impl Mul for u8 { + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + self * rhs + } +} + +impl Mul for usize { + type Output = Self; + + fn mul(self, rhs: Self) -> Self::Output { + self * rhs + } +} + +#[lang = "add"] +pub trait Add<RHS = Self> { + type Output; + + fn add(self, rhs: RHS) -> Self::Output; +} + +impl Add for u8 { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +impl Add for i8 { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +impl Add for usize { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + self + rhs + } +} + +#[lang = "sub"] +pub trait Sub<RHS = Self> { + type Output; + + fn sub(self, rhs: RHS) -> Self::Output; +} + +impl Sub for usize { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for u8 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for i8 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +impl Sub for i16 { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + self - rhs + } +} + +#[lang = "rem"] +pub trait Rem<RHS = Self> { + type Output; + + fn rem(self, rhs: RHS) -> Self::Output; +} + +impl Rem for usize { + type Output = Self; + + fn rem(self, rhs: Self) -> Self { + self % rhs + } +} + +#[lang = "bitor"] +pub trait BitOr<RHS = Self> { + type Output; + + #[must_use] + fn bitor(self, rhs: RHS) -> Self::Output; +} + +impl BitOr for bool { + type Output = bool; + + fn bitor(self, rhs: bool) -> bool { + self | rhs + } +} + +impl<'a> BitOr<bool> for &'a bool { + type Output = bool; + + fn bitor(self, rhs: bool) -> bool { + *self | rhs + } +} + +#[lang = "eq"] +pub trait PartialEq<Rhs: ?Sized = Self> { + fn eq(&self, other: &Rhs) -> bool; + fn ne(&self, other: &Rhs) -> bool; +} + +impl PartialEq for u8 { + fn eq(&self, other: &u8) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u8) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u16 { + fn eq(&self, other: &u16) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u16) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for u32 { + fn eq(&self, other: &u32) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u32) -> bool { + (*self) != (*other) + } +} + + +impl PartialEq for u64 { + fn eq(&self, other: &u64) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &u64) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for usize { + fn eq(&self, other: &usize) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &usize) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for i8 { + fn eq(&self, other: &i8) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &i8) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for i32 { + fn eq(&self, other: &i32) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &i32) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for isize { + fn eq(&self, other: &isize) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &isize) -> bool { + (*self) != (*other) + } +} + +impl PartialEq for char { + fn eq(&self, other: &char) -> bool { + (*self) == (*other) + } + fn ne(&self, other: &char) -> bool { + (*self) != (*other) + } +} + +impl<T: ?Sized> PartialEq for *const T { + fn eq(&self, other: &*const T) -> bool { + *self == *other + } + fn ne(&self, other: &*const T) -> bool { + *self != *other + } +} + +impl <T: PartialEq> PartialEq for Option<T> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Some(lhs), Some(rhs)) => *lhs == *rhs, + (None, None) => true, + _ => false, + } + } + + fn ne(&self, other: &Self) -> bool { + match (self, other) { + (Some(lhs), Some(rhs)) => *lhs != *rhs, + (None, None) => false, + _ => true, + } + } +} + +#[lang = "neg"] +pub trait Neg { + type Output; + + fn neg(self) -> Self::Output; +} + +impl Neg for i8 { + type Output = i8; + + fn neg(self) -> i8 { + -self + } +} + +impl Neg for i16 { + type Output = i16; + + fn neg(self) -> i16 { + self + } +} + +impl Neg for isize { + type Output = isize; + + fn neg(self) -> isize { + -self + } +} + +impl Neg for f32 { + type Output = f32; + + fn neg(self) -> f32 { + -self + } +} + +pub enum Option<T> { + Some(T), + None, +} + +pub use Option::*; + +#[lang = "phantom_data"] +pub struct PhantomData<T: ?Sized>; + +#[lang = "fn_once"] +#[rustc_paren_sugar] +pub trait FnOnce<Args> { + #[lang = "fn_once_output"] + type Output; + + extern "rust-call" fn call_once(self, args: Args) -> Self::Output; +} + +#[lang = "fn_mut"] +#[rustc_paren_sugar] +pub trait FnMut<Args>: FnOnce<Args> { + extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output; +} + +#[lang = "panic"] +#[track_caller] +pub fn panic(_msg: &str) -> ! { + unsafe { + libc::puts("Panicking\n\0" as *const str as *const i8); + intrinsics::abort(); + } +} + +#[lang = "panic_bounds_check"] +#[track_caller] +fn panic_bounds_check(index: usize, len: usize) -> ! { + unsafe { + libc::printf("index out of bounds: the len is %d but the index is %d\n\0" as *const str as *const i8, len, index); + intrinsics::abort(); + } +} + +#[lang = "eh_personality"] +fn eh_personality() -> ! { + loop {} +} + +#[lang = "drop_in_place"] +#[allow(unconditional_recursion)] +pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) { + // Code here does not matter - this is replaced by the + // real drop glue by the compiler. + drop_in_place(to_drop); +} + +#[lang = "deref"] +pub trait Deref { + type Target: ?Sized; + + fn deref(&self) -> &Self::Target; +} + +#[lang = "owned_box"] +pub struct Box<T: ?Sized>(*mut T); + +impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {} + +impl<T: ?Sized> Drop for Box<T> { + fn drop(&mut self) { + // drop is currently performed by compiler. + } +} + +impl<T> Deref for Box<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &**self + } +} + +#[lang = "exchange_malloc"] +unsafe fn allocate(size: usize, _align: usize) -> *mut u8 { + libc::malloc(size) +} + +#[lang = "box_free"] +unsafe fn box_free<T: ?Sized>(ptr: *mut T) { + libc::free(ptr as *mut u8); +} + +#[lang = "drop"] +pub trait Drop { + fn drop(&mut self); +} + +#[lang = "manually_drop"] +#[repr(transparent)] +pub struct ManuallyDrop<T: ?Sized> { + pub value: T, +} + +#[lang = "maybe_uninit"] +#[repr(transparent)] +pub union MaybeUninit<T> { + pub uninit: (), + pub value: ManuallyDrop<T>, +} + +pub mod intrinsics { + extern "rust-intrinsic" { + pub fn abort() -> !; + pub fn size_of<T>() -> usize; + pub fn size_of_val<T: ?::Sized>(val: *const T) -> usize; + pub fn min_align_of<T>() -> usize; + pub fn min_align_of_val<T: ?::Sized>(val: *const T) -> usize; + pub fn copy<T>(src: *const T, dst: *mut T, count: usize); + pub fn transmute<T, U>(e: T) -> U; + pub fn ctlz_nonzero<T>(x: T) -> T; + pub fn needs_drop<T>() -> bool; + pub fn bitreverse<T>(x: T) -> T; + pub fn bswap<T>(x: T) -> T; + pub fn write_bytes<T>(dst: *mut T, val: u8, count: usize); + } +} + +pub mod libc { + #[cfg_attr(not(windows), link(name = "c"))] + #[cfg_attr(windows, link(name = "msvcrt"))] + extern "C" { + pub fn puts(s: *const i8) -> i32; + pub fn printf(format: *const i8, ...) -> i32; + pub fn malloc(size: usize) -> *mut u8; + pub fn free(ptr: *mut u8); + pub fn memcpy(dst: *mut u8, src: *const u8, size: usize); + pub fn memmove(dst: *mut u8, src: *const u8, size: usize); + pub fn strncpy(dst: *mut u8, src: *const u8, size: usize); + } +} + +#[lang = "index"] +pub trait Index<Idx: ?Sized> { + type Output: ?Sized; + fn index(&self, index: Idx) -> &Self::Output; +} + +impl<T> Index<usize> for [T; 3] { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self[index] + } +} + +impl<T> Index<usize> for [T] { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self[index] + } +} + +extern { + type VaListImpl; +} + +#[lang = "va_list"] +#[repr(transparent)] +pub struct VaList<'a>(&'a mut VaListImpl); + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro stringify($($t:tt)*) { /* compiler built-in */ } + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro file() { /* compiler built-in */ } + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro line() { /* compiler built-in */ } + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro cfg() { /* compiler built-in */ } + +#[rustc_builtin_macro] +#[rustc_macro_transparency = "semitransparent"] +pub macro global_asm() { /* compiler built-in */ } + +pub static A_STATIC: u8 = 42; + +#[lang = "panic_location"] +struct PanicLocation { + file: &'static str, + line: u32, + column: u32, +} + +#[no_mangle] +pub fn get_tls() -> u8 { + #[thread_local] + static A: u8 = 42; + + A +} diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs new file mode 100644 index 00000000000..376056e1938 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs @@ -0,0 +1,448 @@ +#![feature( + no_core, start, lang_items, box_syntax, never_type, linkage, + extern_types, thread_local +)] +#![no_core] +#![allow(dead_code, non_camel_case_types)] + +extern crate mini_core; + +use mini_core::*; +use mini_core::libc::*; + +unsafe extern "C" fn my_puts(s: *const i8) { + puts(s); +} + +#[lang = "termination"] +trait Termination { + fn report(self) -> i32; +} + +impl Termination for () { + fn report(self) -> i32 { + unsafe { + NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44 + *NUM_REF as i32 + } + } +} + +trait SomeTrait { + fn object_safe(&self); +} + +impl SomeTrait for &'static str { + fn object_safe(&self) { + unsafe { + puts(*self as *const str as *const i8); + } + } +} + +struct NoisyDrop { + text: &'static str, + inner: NoisyDropInner, +} + +struct NoisyDropInner; + +impl Drop for NoisyDrop { + fn drop(&mut self) { + unsafe { + puts(self.text as *const str as *const i8); + } + } +} + +impl Drop for NoisyDropInner { + fn drop(&mut self) { + unsafe { + puts("Inner got dropped!\0" as *const str as *const i8); + } + } +} + +impl SomeTrait for NoisyDrop { + fn object_safe(&self) {} +} + +enum Ordering { + Less = -1, + Equal = 0, + Greater = 1, +} + +#[lang = "start"] +fn start<T: Termination + 'static>( + main: fn() -> T, + argc: isize, + argv: *const *const u8, +) -> isize { + if argc == 3 { + unsafe { puts(*argv as *const i8); } + unsafe { puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8)); } + unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); } + } + + main().report(); + 0 +} + +static mut NUM: u8 = 6 * 7; +static NUM_REF: &'static u8 = unsafe { &NUM }; + +macro_rules! assert { + ($e:expr) => { + if !$e { + panic(stringify!(! $e)); + } + }; +} + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + } +} + +struct Unique<T: ?Sized> { + pointer: *const T, + _marker: PhantomData<T>, +} + +impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {} + +unsafe fn zeroed<T>() -> T { + let mut uninit = MaybeUninit { uninit: () }; + intrinsics::write_bytes(&mut uninit.value.value as *mut T, 0, 1); + uninit.value.value +} + +fn take_f32(_f: f32) {} +fn take_unique(_u: Unique<()>) {} + +fn return_u128_pair() -> (u128, u128) { + (0, 0) +} + +fn call_return_u128_pair() { + return_u128_pair(); +} + +fn main() { + take_unique(Unique { + pointer: 0 as *const (), + _marker: PhantomData, + }); + take_f32(0.1); + + call_return_u128_pair(); + + let slice = &[0, 1] as &[i32]; + let slice_ptr = slice as *const [i32] as *const i32; + + assert_eq!(slice_ptr as usize % 4, 0); + + //return; + + unsafe { + printf("Hello %s\n\0" as *const str as *const i8, "printf\0" as *const str as *const i8); + + let hello: &[u8] = b"Hello\0" as &[u8; 6]; + let ptr: *const i8 = hello as *const [u8] as *const i8; + puts(ptr); + + let world: Box<&str> = box "World!\0"; + puts(*world as *const str as *const i8); + world as Box<dyn SomeTrait>; + + assert_eq!(intrinsics::bitreverse(0b10101000u8), 0b00010101u8); + + assert_eq!(intrinsics::bswap(0xabu8), 0xabu8); + assert_eq!(intrinsics::bswap(0xddccu16), 0xccddu16); + assert_eq!(intrinsics::bswap(0xffee_ddccu32), 0xccdd_eeffu32); + assert_eq!(intrinsics::bswap(0x1234_5678_ffee_ddccu64), 0xccdd_eeff_7856_3412u64); + + assert_eq!(intrinsics::size_of_val(hello) as u8, 6); + + let chars = &['C', 'h', 'a', 'r', 's']; + let chars = chars as &[char]; + assert_eq!(intrinsics::size_of_val(chars) as u8, 4 * 5); + + let a: &dyn SomeTrait = &"abc\0"; + a.object_safe(); + + assert_eq!(intrinsics::size_of_val(a) as u8, 16); + assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4); + + assert_eq!(intrinsics::min_align_of::<u16>() as u8, 2); + assert_eq!(intrinsics::min_align_of_val(&a) as u8, intrinsics::min_align_of::<&str>() as u8); + + assert!(!intrinsics::needs_drop::<u8>()); + assert!(intrinsics::needs_drop::<NoisyDrop>()); + + Unique { + pointer: 0 as *const &str, + _marker: PhantomData, + } as Unique<dyn SomeTrait>; + + struct MyDst<T: ?Sized>(T); + + intrinsics::size_of_val(&MyDst([0u8; 4]) as &MyDst<[u8]>); + + struct Foo { + x: u8, + y: !, + } + + unsafe fn uninitialized<T>() -> T { + MaybeUninit { uninit: () }.value.value + } + + zeroed::<(u8, u8)>(); + #[allow(unreachable_code)] + { + if false { + zeroed::<!>(); + zeroed::<Foo>(); + uninitialized::<Foo>(); + } + } + } + + let _ = box NoisyDrop { + text: "Boxed outer got dropped!\0", + inner: NoisyDropInner, + } as Box<dyn SomeTrait>; + + const FUNC_REF: Option<fn()> = Some(main); + match FUNC_REF { + Some(_) => {}, + None => assert!(false), + } + + match Ordering::Less { + Ordering::Less => {}, + _ => assert!(false), + } + + [NoisyDropInner, NoisyDropInner]; + + let x = &[0u32, 42u32] as &[u32]; + match x { + [] => assert_eq!(0u32, 1), + [_, ref y @ ..] => assert_eq!(&x[1] as *const u32 as usize, &y[0] as *const u32 as usize), + } + + assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42); + + #[cfg(not(jit))] + { + extern { + #[linkage = "extern_weak"] + static ABC: *const u8; + } + + { + extern { + #[linkage = "extern_weak"] + static ABC: *const u8; + } + } + + unsafe { assert_eq!(ABC as usize, 0); } + } + + &mut (|| Some(0 as *const ())) as &mut dyn FnMut() -> Option<*const ()>; + + let f = 1000.0; + assert_eq!(f as u8, 255); + let f2 = -1000.0; + assert_eq!(f2 as i8, -128); + assert_eq!(f2 as u8, 0); + + static ANOTHER_STATIC: &u8 = &A_STATIC; + assert_eq!(*ANOTHER_STATIC, 42); + + check_niche_behavior(); + + extern "C" { + type ExternType; + } + + struct ExternTypeWrapper { + _a: ExternType, + } + + let nullptr = 0 as *const (); + let extern_nullptr = nullptr as *const ExternTypeWrapper; + extern_nullptr as *const (); + let slice_ptr = &[] as *const [u8]; + slice_ptr as *const u8; + + let repeat = [Some(42); 2]; + assert_eq!(repeat[0], Some(42)); + assert_eq!(repeat[1], Some(42)); + + #[cfg(not(jit))] + test_tls(); + + #[cfg(all(not(jit), target_os = "linux"))] + unsafe { + global_asm_test(); + } +} + +#[cfg(all(not(jit), target_os = "linux"))] +extern "C" { + fn global_asm_test(); +} + +#[cfg(all(not(jit), target_os = "linux"))] +global_asm! { + " + .global global_asm_test + global_asm_test: + // comment that would normally be removed by LLVM + ret + " +} + +#[repr(C)] +enum c_void { + _1, + _2, +} + +type c_int = i32; +type c_ulong = u64; + +type pthread_t = c_ulong; + +#[repr(C)] +struct pthread_attr_t { + __size: [u64; 7], +} + +#[link(name = "pthread")] +extern "C" { + fn pthread_attr_init(attr: *mut pthread_attr_t) -> c_int; + + fn pthread_create( + native: *mut pthread_t, + attr: *const pthread_attr_t, + f: extern "C" fn(_: *mut c_void) -> *mut c_void, + value: *mut c_void + ) -> c_int; + + fn pthread_join( + native: pthread_t, + value: *mut *mut c_void + ) -> c_int; +} + +#[thread_local] +#[cfg(not(jit))] +static mut TLS: u8 = 42; + +#[cfg(not(jit))] +extern "C" fn mutate_tls(_: *mut c_void) -> *mut c_void { + unsafe { TLS = 0; } + 0 as *mut c_void +} + +#[cfg(not(jit))] +fn test_tls() { + unsafe { + let mut attr: pthread_attr_t = zeroed(); + let mut thread: pthread_t = 0; + + assert_eq!(TLS, 42); + + if pthread_attr_init(&mut attr) != 0 { + assert!(false); + } + + if pthread_create(&mut thread, &attr, mutate_tls, 0 as *mut c_void) != 0 { + assert!(false); + } + + let mut res = 0 as *mut c_void; + pthread_join(thread, &mut res); + + // TLS of main thread must not have been changed by the other thread. + assert_eq!(TLS, 42); + + puts("TLS works!\n\0" as *const str as *const i8); + } +} + +// Copied ui/issues/issue-61696.rs + +pub enum Infallible {} + +// The check that the `bool` field of `V1` is encoding a "niche variant" +// (i.e. not `V1`, so `V3` or `V4`) used to be mathematically incorrect, +// causing valid `V1` values to be interpreted as other variants. +pub enum E1 { + V1 { f: bool }, + V2 { f: Infallible }, + V3, + V4, +} + +// Computing the discriminant used to be done using the niche type (here `u8`, +// from the `bool` field of `V1`), overflowing for variants with large enough +// indices (`V3` and `V4`), causing them to be interpreted as other variants. +pub enum E2<X> { + V1 { f: bool }, + + /*_00*/ _01(X), _02(X), _03(X), _04(X), _05(X), _06(X), _07(X), + _08(X), _09(X), _0A(X), _0B(X), _0C(X), _0D(X), _0E(X), _0F(X), + _10(X), _11(X), _12(X), _13(X), _14(X), _15(X), _16(X), _17(X), + _18(X), _19(X), _1A(X), _1B(X), _1C(X), _1D(X), _1E(X), _1F(X), + _20(X), _21(X), _22(X), _23(X), _24(X), _25(X), _26(X), _27(X), + _28(X), _29(X), _2A(X), _2B(X), _2C(X), _2D(X), _2E(X), _2F(X), + _30(X), _31(X), _32(X), _33(X), _34(X), _35(X), _36(X), _37(X), + _38(X), _39(X), _3A(X), _3B(X), _3C(X), _3D(X), _3E(X), _3F(X), + _40(X), _41(X), _42(X), _43(X), _44(X), _45(X), _46(X), _47(X), + _48(X), _49(X), _4A(X), _4B(X), _4C(X), _4D(X), _4E(X), _4F(X), + _50(X), _51(X), _52(X), _53(X), _54(X), _55(X), _56(X), _57(X), + _58(X), _59(X), _5A(X), _5B(X), _5C(X), _5D(X), _5E(X), _5F(X), + _60(X), _61(X), _62(X), _63(X), _64(X), _65(X), _66(X), _67(X), + _68(X), _69(X), _6A(X), _6B(X), _6C(X), _6D(X), _6E(X), _6F(X), + _70(X), _71(X), _72(X), _73(X), _74(X), _75(X), _76(X), _77(X), + _78(X), _79(X), _7A(X), _7B(X), _7C(X), _7D(X), _7E(X), _7F(X), + _80(X), _81(X), _82(X), _83(X), _84(X), _85(X), _86(X), _87(X), + _88(X), _89(X), _8A(X), _8B(X), _8C(X), _8D(X), _8E(X), _8F(X), + _90(X), _91(X), _92(X), _93(X), _94(X), _95(X), _96(X), _97(X), + _98(X), _99(X), _9A(X), _9B(X), _9C(X), _9D(X), _9E(X), _9F(X), + _A0(X), _A1(X), _A2(X), _A3(X), _A4(X), _A5(X), _A6(X), _A7(X), + _A8(X), _A9(X), _AA(X), _AB(X), _AC(X), _AD(X), _AE(X), _AF(X), + _B0(X), _B1(X), _B2(X), _B3(X), _B4(X), _B5(X), _B6(X), _B7(X), + _B8(X), _B9(X), _BA(X), _BB(X), _BC(X), _BD(X), _BE(X), _BF(X), + _C0(X), _C1(X), _C2(X), _C3(X), _C4(X), _C5(X), _C6(X), _C7(X), + _C8(X), _C9(X), _CA(X), _CB(X), _CC(X), _CD(X), _CE(X), _CF(X), + _D0(X), _D1(X), _D2(X), _D3(X), _D4(X), _D5(X), _D6(X), _D7(X), + _D8(X), _D9(X), _DA(X), _DB(X), _DC(X), _DD(X), _DE(X), _DF(X), + _E0(X), _E1(X), _E2(X), _E3(X), _E4(X), _E5(X), _E6(X), _E7(X), + _E8(X), _E9(X), _EA(X), _EB(X), _EC(X), _ED(X), _EE(X), _EF(X), + _F0(X), _F1(X), _F2(X), _F3(X), _F4(X), _F5(X), _F6(X), _F7(X), + _F8(X), _F9(X), _FA(X), _FB(X), _FC(X), _FD(X), _FE(X), _FF(X), + + V3, + V4, +} + +fn check_niche_behavior () { + if let E1::V2 { .. } = (E1::V1 { f: true }) { + intrinsics::abort(); + } + + if let E2::V1 { .. } = E2::V3::<Infallible> { + intrinsics::abort(); + } +} diff --git a/compiler/rustc_codegen_cranelift/example/mod_bench.rs b/compiler/rustc_codegen_cranelift/example/mod_bench.rs new file mode 100644 index 00000000000..bc652213623 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/mod_bench.rs @@ -0,0 +1,35 @@ +#![feature(start, box_syntax, core_intrinsics, lang_items)] +#![no_std] + +#[link(name = "c")] +extern {} + +#[panic_handler] +fn panic_handler(_: &core::panic::PanicInfo) -> ! { + core::intrinsics::abort(); +} + +#[lang="eh_personality"] +fn eh_personality(){} + +// Required for rustc_codegen_llvm +#[no_mangle] +unsafe extern "C" fn _Unwind_Resume() { + core::intrinsics::unreachable(); +} + +#[start] +fn main(_argc: isize, _argv: *const *const u8) -> isize { + for i in 2..10_000_000 { + black_box((i + 1) % i); + } + + 0 +} + +#[inline(never)] +fn black_box(i: u32) { + if i != 1 { + core::intrinsics::abort(); + } +} diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs new file mode 100644 index 00000000000..079b4299049 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -0,0 +1,343 @@ +#![feature(core_intrinsics, generators, generator_trait, is_sorted)] + +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::io::Write; +use std::ops::Generator; + +fn main() { + println!("{:?}", std::env::args().collect::<Vec<_>>()); + + let mutex = std::sync::Mutex::new(()); + let _guard = mutex.lock().unwrap(); + + let _ = ::std::iter::repeat('a' as u8).take(10).collect::<Vec<_>>(); + let stderr = ::std::io::stderr(); + let mut stderr = stderr.lock(); + + std::thread::spawn(move || { + println!("Hello from another thread!"); + }); + + writeln!(stderr, "some {} text", "<unknown>").unwrap(); + + let _ = std::process::Command::new("true").env("c", "d").spawn(); + + println!("cargo:rustc-link-lib=z"); + + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| {}); + + let _eq = LoopState::Continue(()) == LoopState::Break(()); + + // Make sure ByValPair values with differently sized components are correctly passed + map(None::<(u8, Box<Instruction>)>); + + println!("{}", 2.3f32.exp()); + println!("{}", 2.3f32.exp2()); + println!("{}", 2.3f32.abs()); + println!("{}", 2.3f32.sqrt()); + println!("{}", 2.3f32.floor()); + println!("{}", 2.3f32.ceil()); + println!("{}", 2.3f32.min(1.0)); + println!("{}", 2.3f32.max(1.0)); + println!("{}", 2.3f32.powi(2)); + println!("{}", 2.3f32.log2()); + assert_eq!(2.3f32.copysign(-1.0), -2.3f32); + println!("{}", 2.3f32.powf(2.0)); + + assert_eq!(-128i8, (-128i8).saturating_sub(1)); + assert_eq!(127i8, 127i8.saturating_sub(-128)); + assert_eq!(-128i8, (-128i8).saturating_add(-128)); + assert_eq!(127i8, 127i8.saturating_add(1)); + + assert_eq!(0b0000000000000000000000000010000010000000000000000000000000000000_0000000000100000000000000000000000001000000000000100000000000000u128.leading_zeros(), 26); + assert_eq!(0b0000000000000000000000000010000000000000000000000000000000000000_0000000000000000000000000000000000001000000000000000000010000000u128.trailing_zeros(), 7); + + let _d = 0i128.checked_div(2i128); + let _d = 0u128.checked_div(2u128); + assert_eq!(1u128 + 2, 3); + + assert_eq!(0b100010000000000000000000000000000u128 >> 10, 0b10001000000000000000000u128); + assert_eq!(0xFEDCBA987654321123456789ABCDEFu128 >> 64, 0xFEDCBA98765432u128); + assert_eq!(0xFEDCBA987654321123456789ABCDEFu128 as i128 >> 64, 0xFEDCBA98765432i128); + + let tmp = 353985398u128; + assert_eq!(tmp * 932490u128, 330087843781020u128); + + let tmp = -0x1234_5678_9ABC_DEF0i64; + assert_eq!(tmp as i128, -0x1234_5678_9ABC_DEF0i128); + + // Check that all u/i128 <-> float casts work correctly. + let houndred_u128 = 100u128; + let houndred_i128 = 100i128; + let houndred_f32 = 100.0f32; + let houndred_f64 = 100.0f64; + assert_eq!(houndred_u128 as f32, 100.0); + assert_eq!(houndred_u128 as f64, 100.0); + assert_eq!(houndred_f32 as u128, 100); + assert_eq!(houndred_f64 as u128, 100); + assert_eq!(houndred_i128 as f32, 100.0); + assert_eq!(houndred_i128 as f64, 100.0); + assert_eq!(houndred_f32 as i128, 100); + assert_eq!(houndred_f64 as i128, 100); + + // Test signed 128bit comparing + let max = usize::MAX as i128; + if 100i128 < 0i128 || 100i128 > max { + panic!(); + } + + test_checked_mul(); + + let _a = 1u32 << 2u8; + + let empty: [i32; 0] = []; + assert!(empty.is_sorted()); + + println!("{:?}", std::intrinsics::caller_location()); + + #[cfg(target_arch = "x86_64")] + unsafe { + test_simd(); + } + + Box::pin(move |mut _task_context| { + yield (); + }).as_mut().resume(0); + + #[derive(Copy, Clone)] + enum Nums { + NegOne = -1, + } + + let kind = Nums::NegOne; + assert_eq!(-1i128, kind as i128); + + let options = [1u128]; + match options[0] { + 1 => (), + 0 => loop {}, + v => panic(v), + }; +} + +fn panic(_: u128) { + panic!(); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_simd() { + assert!(is_x86_feature_detected!("sse2")); + + let x = _mm_setzero_si128(); + let y = _mm_set1_epi16(7); + let or = _mm_or_si128(x, y); + let cmp_eq = _mm_cmpeq_epi8(y, y); + let cmp_lt = _mm_cmplt_epi8(y, y); + + assert_eq!(std::mem::transmute::<_, [u16; 8]>(or), [7, 7, 7, 7, 7, 7, 7, 7]); + assert_eq!(std::mem::transmute::<_, [u16; 8]>(cmp_eq), [0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff]); + assert_eq!(std::mem::transmute::<_, [u16; 8]>(cmp_lt), [0, 0, 0, 0, 0, 0, 0, 0]); + + test_mm_slli_si128(); + test_mm_movemask_epi8(); + test_mm256_movemask_epi8(); + test_mm_add_epi8(); + test_mm_add_pd(); + test_mm_cvtepi8_epi16(); + test_mm_cvtsi128_si64(); + + test_mm_extract_epi8(); + test_mm_insert_epi16(); + + let mask1 = _mm_movemask_epi8(dbg!(_mm_setr_epi8(255u8 as i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))); + assert_eq!(mask1, 1); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_slli_si128() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 1); + let e = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m128i(r, e); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 15); + let e = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); + assert_eq_m128i(r, e); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, 16); + assert_eq_m128i(r, _mm_set1_epi8(0)); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, -1); + assert_eq_m128i(_mm_set1_epi8(0), r); + + #[rustfmt::skip] + let a = _mm_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ); + let r = _mm_slli_si128(a, -0x80000000); + assert_eq_m128i(r, _mm_set1_epi8(0)); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_movemask_epi8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 0b1000_0000u8 as i8, 0b0, 0b1000_0000u8 as i8, 0b01, + 0b0101, 0b1111_0000u8 as i8, 0, 0, + 0, 0, 0b1111_0000u8 as i8, 0b0101, + 0b01, 0b1000_0000u8 as i8, 0b0, 0b1000_0000u8 as i8, + ); + let r = _mm_movemask_epi8(a); + assert_eq!(r, 0b10100100_00100101); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn test_mm256_movemask_epi8() { + let a = _mm256_set1_epi8(-1); + let r = _mm256_movemask_epi8(a); + let e = -1; + assert_eq!(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_add_epi8() { + let a = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + #[rustfmt::skip] + let b = _mm_setr_epi8( + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + ); + let r = _mm_add_epi8(a, b); + #[rustfmt::skip] + let e = _mm_setr_epi8( + 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, + ); + assert_eq_m128i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_add_pd() { + let a = _mm_setr_pd(1.0, 2.0); + let b = _mm_setr_pd(5.0, 10.0); + let r = _mm_add_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(6.0, 12.0)); +} + +#[cfg(target_arch = "x86_64")] +fn assert_eq_m128i(x: std::arch::x86_64::__m128i, y: std::arch::x86_64::__m128i) { + unsafe { + assert_eq!(std::mem::transmute::<_, [u8; 16]>(x), std::mem::transmute::<_, [u8; 16]>(y)); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_cvtsi128_si64() { + let r = _mm_cvtsi128_si64(std::mem::transmute::<[i64; 2], _>([5, 0])); + assert_eq!(r, 5); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse4.1")] +unsafe fn test_mm_cvtepi8_epi16() { + let a = _mm_set1_epi8(10); + let r = _mm_cvtepi8_epi16(a); + let e = _mm_set1_epi16(10); + assert_eq_m128i(r, e); + let a = _mm_set1_epi8(-10); + let r = _mm_cvtepi8_epi16(a); + let e = _mm_set1_epi16(-10); + assert_eq_m128i(r, e); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse4.1")] +unsafe fn test_mm_extract_epi8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + -1, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15 + ); + let r1 = _mm_extract_epi8(a, 0); + let r2 = _mm_extract_epi8(a, 19); + assert_eq!(r1, 0xFF); + assert_eq!(r2, 3); +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "sse2")] +unsafe fn test_mm_insert_epi16() { + let a = _mm_setr_epi16(0, 1, 2, 3, 4, 5, 6, 7); + let r = _mm_insert_epi16(a, 9, 0); + let e = _mm_setr_epi16(9, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(r, e); +} + +fn test_checked_mul() { + let u: Option<u8> = u8::from_str_radix("1000", 10).ok(); + assert_eq!(u, None); + + assert_eq!(1u8.checked_mul(255u8), Some(255u8)); + assert_eq!(255u8.checked_mul(255u8), None); + assert_eq!(1i8.checked_mul(127i8), Some(127i8)); + assert_eq!(127i8.checked_mul(127i8), None); + assert_eq!((-1i8).checked_mul(-127i8), Some(127i8)); + assert_eq!(1i8.checked_mul(-128i8), Some(-128i8)); + assert_eq!((-128i8).checked_mul(-128i8), None); + + assert_eq!(1u64.checked_mul(u64::max_value()), Some(u64::max_value())); + assert_eq!(u64::max_value().checked_mul(u64::max_value()), None); + assert_eq!(1i64.checked_mul(i64::max_value()), Some(i64::max_value())); + assert_eq!(i64::max_value().checked_mul(i64::max_value()), None); + assert_eq!((-1i64).checked_mul(i64::min_value() + 1), Some(i64::max_value())); + assert_eq!(1i64.checked_mul(i64::min_value()), Some(i64::min_value())); + assert_eq!(i64::min_value().checked_mul(i64::min_value()), None); +} + +#[derive(PartialEq)] +enum LoopState { + Continue(()), + Break(()) +} + +pub enum Instruction { + Increment, + Loop, +} + +fn map(a: Option<(u8, Box<Instruction>)>) -> Option<Box<Instruction>> { + match a { + None => None, + Some((_, instr)) => Some(instr), + } +} diff --git a/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs b/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs new file mode 100644 index 00000000000..2cb84786f56 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/subslice-patterns-const-eval.rs @@ -0,0 +1,97 @@ +// Based on https://github.com/rust-lang/rust/blob/c5840f9d252c2f5cc16698dbf385a29c5de3ca07/src/test/ui/array-slice-vec/subslice-patterns-const-eval-match.rs + +// Test that array subslice patterns are correctly handled in const evaluation. + +// run-pass + +#[derive(PartialEq, Debug, Clone)] +struct N(u8); + +#[derive(PartialEq, Debug, Clone)] +struct Z; + +macro_rules! n { + ($($e:expr),* $(,)?) => { + [$(N($e)),*] + } +} + +// This macro has an unused variable so that it can be repeated base on the +// number of times a repeated variable (`$e` in `z`) occurs. +macro_rules! zed { + ($e:expr) => { Z } +} + +macro_rules! z { + ($($e:expr),* $(,)?) => { + [$(zed!($e)),*] + } +} + +// Compare constant evaluation and runtime evaluation of a given expression. +macro_rules! compare_evaluation { + ($e:expr, $t:ty $(,)?) => {{ + const CONST_EVAL: $t = $e; + const fn const_eval() -> $t { $e } + static CONST_EVAL2: $t = const_eval(); + let runtime_eval = $e; + assert_eq!(CONST_EVAL, runtime_eval); + assert_eq!(CONST_EVAL2, runtime_eval); + }} +} + +// Repeat `$test`, substituting the given macro variables with the given +// identifiers. +// +// For example: +// +// repeat! { +// ($name); X; Y: +// struct $name; +// } +// +// Expands to: +// +// struct X; struct Y; +// +// This is used to repeat the tests using both the `N` and `Z` +// types. +macro_rules! repeat { + (($($dollar:tt $placeholder:ident)*); $($($values:ident),+);*: $($test:tt)*) => { + macro_rules! single { + ($($dollar $placeholder:ident),*) => { $($test)* } + } + $(single!($($values),+);)* + } +} + +fn main() { + repeat! { + ($arr $Ty); n, N; z, Z: + compare_evaluation!({ let [_, x @ .., _] = $arr!(1, 2, 3, 4); x }, [$Ty; 2]); + compare_evaluation!({ let [_, ref x @ .., _] = $arr!(1, 2, 3, 4); x }, &'static [$Ty; 2]); + compare_evaluation!({ let [_, x @ .., _] = &$arr!(1, 2, 3, 4); x }, &'static [$Ty; 2]); + + compare_evaluation!({ let [_, _, x @ .., _, _] = $arr!(1, 2, 3, 4); x }, [$Ty; 0]); + compare_evaluation!( + { let [_, _, ref x @ .., _, _] = $arr!(1, 2, 3, 4); x }, + &'static [$Ty; 0], + ); + compare_evaluation!( + { let [_, _, x @ .., _, _] = &$arr!(1, 2, 3, 4); x }, + &'static [$Ty; 0], + ); + + compare_evaluation!({ let [_, .., x] = $arr!(1, 2, 3, 4); x }, $Ty); + compare_evaluation!({ let [_, .., ref x] = $arr!(1, 2, 3, 4); x }, &'static $Ty); + compare_evaluation!({ let [_, _y @ .., x] = &$arr!(1, 2, 3, 4); x }, &'static $Ty); + } + + compare_evaluation!({ let [_, .., N(x)] = n!(1, 2, 3, 4); x }, u8); + compare_evaluation!({ let [_, .., N(ref x)] = n!(1, 2, 3, 4); x }, &'static u8); + compare_evaluation!({ let [_, .., N(x)] = &n!(1, 2, 3, 4); x }, &'static u8); + + compare_evaluation!({ let [N(x), .., _] = n!(1, 2, 3, 4); x }, u8); + compare_evaluation!({ let [N(ref x), .., _] = n!(1, 2, 3, 4); x }, &'static u8); + compare_evaluation!({ let [N(x), .., _] = &n!(1, 2, 3, 4); x }, &'static u8); +} diff --git a/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs b/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs new file mode 100644 index 00000000000..93bab17e46b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/example/track-caller-attribute.rs @@ -0,0 +1,40 @@ +// Based on https://github.com/anp/rust/blob/175631311716d7dfeceec40d2587cde7142ffa8c/src/test/ui/rfc-2091-track-caller/track-caller-attribute.rs + +// run-pass + +use std::panic::Location; + +#[track_caller] +fn tracked() -> &'static Location<'static> { + Location::caller() +} + +fn nested_intrinsic() -> &'static Location<'static> { + Location::caller() +} + +fn nested_tracked() -> &'static Location<'static> { + tracked() +} + +fn main() { + let location = Location::caller(); + assert_eq!(location.file(), file!()); + assert_eq!(location.line(), 21); + assert_eq!(location.column(), 20); + + let tracked = tracked(); + assert_eq!(tracked.file(), file!()); + assert_eq!(tracked.line(), 26); + assert_eq!(tracked.column(), 19); + + let nested = nested_intrinsic(); + assert_eq!(nested.file(), file!()); + assert_eq!(nested.line(), 13); + assert_eq!(nested.column(), 5); + + let contained = nested_tracked(); + assert_eq!(contained.file(), file!()); + assert_eq!(contained.line(), 17); + assert_eq!(contained.column(), 5); +} diff --git a/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch new file mode 100644 index 00000000000..ee8548783de --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch @@ -0,0 +1,123 @@ +From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Sun, 24 Nov 2019 15:10:23 +0100 +Subject: [PATCH] [core] Disable not compiling tests + +--- + library/core/tests/Cargo.toml | 8 ++++++++ + library/core/tests/num/flt2dec/mod.rs | 1 - + library/core/tests/num/int_macros.rs | 2 ++ + library/core/tests/num/uint_macros.rs | 2 ++ + library/core/tests/ptr.rs | 2 ++ + library/core/tests/slice.rs | 2 ++ + 6 files changed, 16 insertions(+), 1 deletion(-) + create mode 100644 library/core/tests/Cargo.toml + +diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml +new file mode 100644 +index 0000000..46fd999 +--- /dev/null ++++ b/library/core/tests/Cargo.toml +@@ -0,0 +1,8 @@ ++[package] ++name = "core" ++version = "0.0.0" ++edition = "2018" ++ ++[lib] ++name = "coretests" ++path = "lib.rs" +diff --git a/library/core/tests/num/flt2dec/mod.rs b/library/core/tests/num/flt2dec/mod.rs +index a35897e..f0bf645 100644 +--- a/library/core/tests/num/flt2dec/mod.rs ++++ b/library/core/tests/num/flt2dec/mod.rs +@@ -13,7 +13,6 @@ mod strategy { + mod dragon; + mod grisu; + } +-mod random; + + pub fn decode_finite<T: DecodableFloat>(v: T) -> Decoded { + match decode(v).1 { +diff --git a/library/core/tests/num/int_macros.rs b/library/core/tests/num/int_macros.rs +index 0475aeb..9558198 100644 +--- a/library/core/tests/num/int_macros.rs ++++ b/library/core/tests/num/int_macros.rs +@@ -88,6 +88,7 @@ mod tests { + assert_eq!(x.trailing_ones(), 0); + } + ++ /* + #[test] + fn test_rotate() { + assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); +@@ -112,6 +113,7 @@ mod tests { + assert_eq!(B.rotate_left(64), B); + assert_eq!(C.rotate_left(64), C); + } ++ */ + + #[test] + fn test_swap_bytes() { +diff --git a/library/core/tests/num/uint_macros.rs b/library/core/tests/num/uint_macros.rs +index 04ed14f..a6e372e 100644 +--- a/library/core/tests/num/uint_macros.rs ++++ b/library/core/tests/num/uint_macros.rs +@@ -52,6 +52,7 @@ mod tests { + assert_eq!(x.trailing_ones(), 0); + } + ++ /* + #[test] + fn test_rotate() { + assert_eq!(A.rotate_left(6).rotate_right(2).rotate_right(4), A); +@@ -76,6 +77,7 @@ mod tests { + assert_eq!(B.rotate_left(64), B); + assert_eq!(C.rotate_left(64), C); + } ++ */ + + #[test] + fn test_swap_bytes() { +diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs +index 1a6be3a..42dbd59 100644 +--- a/library/core/tests/ptr.rs ++++ b/library/core/tests/ptr.rs +@@ -250,6 +250,7 @@ fn test_unsized_nonnull() { + assert!(ys == zs); + } + ++/* + #[test] + #[allow(warnings)] + // Have a symbol for the test below. It doesn’t need to be an actual variadic function, match the +@@ -289,6 +290,7 @@ fn write_unaligned_drop() { + } + DROPS.with(|d| assert_eq!(*d.borrow(), [0])); + } ++*/ + + #[test] + fn align_offset_zst() { +diff --git a/library/core/tests/slice.rs b/library/core/tests/slice.rs +index 6609bc3..241b497 100644 +--- a/library/core/tests/slice.rs ++++ b/library/core/tests/slice.rs +@@ -1209,6 +1209,7 @@ fn brute_force_rotate_test_1() { + } + } + ++/* + #[test] + #[cfg(not(target_arch = "wasm32"))] + fn sort_unstable() { +@@ -1394,6 +1395,7 @@ fn partition_at_index() { + v.select_nth_unstable(0); + assert!(v == [0xDEADBEEF]); + } ++*/ + + #[test] + #[should_panic(expected = "index 0 greater than length of slice")] +-- +2.21.0 (Apple Git-122) diff --git a/compiler/rustc_codegen_cranelift/patches/0023-core-Ignore-failing-tests.patch b/compiler/rustc_codegen_cranelift/patches/0023-core-Ignore-failing-tests.patch new file mode 100644 index 00000000000..5d2c3049f60 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0023-core-Ignore-failing-tests.patch @@ -0,0 +1,90 @@ +From dd82e95c9de212524e14fc60155de1ae40156dfc Mon Sep 17 00:00:00 2001 +From: bjorn3 <bjorn3@users.noreply.github.com> +Date: Sun, 24 Nov 2019 15:34:06 +0100 +Subject: [PATCH] [core] Ignore failing tests + +--- + library/core/tests/iter.rs | 4 ++++ + library/core/tests/num/bignum.rs | 10 ++++++++++ + library/core/tests/num/mod.rs | 5 +++-- + library/core/tests/time.rs | 1 + + 4 files changed, 18 insertions(+), 2 deletions(-) + +diff --git a/library/core/tests/array.rs b/library/core/tests/array.rs +index 4bc44e9..8e3c7a4 100644 +--- a/library/core/tests/array.rs ++++ b/library/core/tests/array.rs +@@ -242,6 +242,7 @@ fn iterator_drops() { + assert_eq!(i.get(), 5); + } + ++/* + // This test does not work on targets without panic=unwind support. + // To work around this problem, test is marked is should_panic, so it will + // be automagically skipped on unsuitable targets, such as +@@ -283,6 +284,7 @@ fn array_default_impl_avoids_leaks_on_panic() { + assert_eq!(COUNTER.load(Relaxed), 0); + panic!("test succeeded") + } ++*/ + + #[test] + fn empty_array_is_always_default() { +@@ -304,6 +304,7 @@ fn array_map() { + assert_eq!(b, [1, 2, 3]); + } + ++/* + // See note on above test for why `should_panic` is used. + #[test] + #[should_panic(expected = "test succeeded")] +@@ -332,6 +333,7 @@ fn array_map_drop_safety() { + assert_eq!(DROPPED.load(Ordering::SeqCst), num_to_create); + panic!("test succeeded") + } ++*/ + + #[test] + fn cell_allows_array_cycle() { +diff --git a/library/core/tests/num/mod.rs b/library/core/tests/num/mod.rs +index a17c094..5bb11d2 100644 +--- a/library/core/tests/num/mod.rs ++++ b/library/core/tests/num/mod.rs +@@ -651,11 +651,12 @@ macro_rules! test_float { + assert_eq!((9.0 as $fty).min($neginf), $neginf); + assert_eq!(($neginf as $fty).min(-9.0), $neginf); + assert_eq!((-9.0 as $fty).min($neginf), $neginf); +- assert_eq!(($nan as $fty).min(9.0), 9.0); +- assert_eq!(($nan as $fty).min(-9.0), -9.0); +- assert_eq!((9.0 as $fty).min($nan), 9.0); +- assert_eq!((-9.0 as $fty).min($nan), -9.0); +- assert!(($nan as $fty).min($nan).is_nan()); ++ // Cranelift fmin has NaN propagation ++ //assert_eq!(($nan as $fty).min(9.0), 9.0); ++ //assert_eq!(($nan as $fty).min(-9.0), -9.0); ++ //assert_eq!((9.0 as $fty).min($nan), 9.0); ++ //assert_eq!((-9.0 as $fty).min($nan), -9.0); ++ //assert!(($nan as $fty).min($nan).is_nan()); + } + #[test] + fn max() { +@@ -673,11 +674,12 @@ macro_rules! test_float { + assert_eq!((9.0 as $fty).max($neginf), 9.0); + assert_eq!(($neginf as $fty).max(-9.0), -9.0); + assert_eq!((-9.0 as $fty).max($neginf), -9.0); +- assert_eq!(($nan as $fty).max(9.0), 9.0); +- assert_eq!(($nan as $fty).max(-9.0), -9.0); +- assert_eq!((9.0 as $fty).max($nan), 9.0); +- assert_eq!((-9.0 as $fty).max($nan), -9.0); +- assert!(($nan as $fty).max($nan).is_nan()); ++ // Cranelift fmax has NaN propagation ++ //assert_eq!(($nan as $fty).max(9.0), 9.0); ++ //assert_eq!(($nan as $fty).max(-9.0), -9.0); ++ //assert_eq!((9.0 as $fty).max($nan), 9.0); ++ //assert_eq!((-9.0 as $fty).max($nan), -9.0); ++ //assert!(($nan as $fty).max($nan).is_nan()); + } + #[test] + fn rem_euclid() { +-- +2.21.0 (Apple Git-122) diff --git a/compiler/rustc_codegen_cranelift/prepare.sh b/compiler/rustc_codegen_cranelift/prepare.sh new file mode 100755 index 00000000000..87f96f5dcf4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/prepare.sh @@ -0,0 +1,29 @@ +#!/bin/bash --verbose +set -e + +rustup component add rust-src rustc-dev llvm-tools-preview +./build_sysroot/prepare_sysroot_src.sh +cargo install hyperfine || echo "Skipping hyperfine install" + +git clone https://github.com/rust-random/rand.git || echo "rust-random/rand has already been cloned" +pushd rand +git checkout -- . +git checkout 0f933f9c7176e53b2a3c7952ded484e1783f0bf1 +git am ../crate_patches/*-rand-*.patch +popd + +git clone https://github.com/rust-lang/regex.git || echo "rust-lang/regex has already been cloned" +pushd regex +git checkout -- . +git checkout 341f207c1071f7290e3f228c710817c280c8dca1 +popd + +git clone https://github.com/ebobby/simple-raytracer || echo "ebobby/simple-raytracer has already been cloned" +pushd simple-raytracer +git checkout -- . +git checkout 804a7a21b9e673a482797aa289a18ed480e4d813 + +# build with cg_llvm for perf comparison +cargo build +mv target/debug/main raytracer_cg_llvm +popd diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain new file mode 100644 index 00000000000..87e54719fdc --- /dev/null +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -0,0 +1 @@ +nightly-2020-10-26 diff --git a/compiler/rustc_codegen_cranelift/scripts/Readme.md b/compiler/rustc_codegen_cranelift/scripts/Readme.md new file mode 100644 index 00000000000..83cec9c6f36 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/Readme.md @@ -0,0 +1,2 @@ +This directory is for scripts that are either never directly invoked or are not used very often. +Scripts that are frequently used should be kept at the project root. diff --git a/compiler/rustc_codegen_cranelift/scripts/config.sh b/compiler/rustc_codegen_cranelift/scripts/config.sh new file mode 100644 index 00000000000..530b7f242a0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/config.sh @@ -0,0 +1,56 @@ +set -e + +unamestr=`uname` +if [[ "$unamestr" == 'Linux' ]]; then + dylib_ext='so' +elif [[ "$unamestr" == 'Darwin' ]]; then + dylib_ext='dylib' +else + echo "Unsupported os" + exit 1 +fi + +HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") +TARGET_TRIPLE=$HOST_TRIPLE +#TARGET_TRIPLE="x86_64-pc-windows-gnu" +#TARGET_TRIPLE="aarch64-unknown-linux-gnu" + +linker='' +RUN_WRAPPER='' +export JIT_SUPPORTED=1 +if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then + export JIT_SUPPORTED=0 + if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then + # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. + linker='-Clinker=aarch64-linux-gnu-gcc' + RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu' + elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then + # We are cross-compiling for Windows. Run tests in wine. + RUN_WRAPPER='wine' + else + echo "Unknown non-native platform" + fi +fi + +if echo "$RUSTC_WRAPPER" | grep sccache; then +echo +echo -e "\x1b[1;93m=== Warning: Unset RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m" +echo +export RUSTC_WRAPPER= +fi + +export RUSTC=$(pwd)/"target/"$CHANNEL"/cg_clif" +export RUSTFLAGS=$linker +export RUSTDOCFLAGS=$linker' -Ztrim-diagnostic-paths=no -Cpanic=abort -Zpanic-abort-tests '\ +'-Zcodegen-backend='$(pwd)'/target/'$CHANNEL'/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$(pwd)'/build_sysroot/sysroot' + +# FIXME remove once the atomic shim is gone +if [[ `uname` == 'Darwin' ]]; then + export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" +fi + +export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/"$TARGET_TRIPLE"/lib:\ +$(pwd)/target/"$CHANNEL":$(rustc --print sysroot)/lib" +export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH + +export CG_CLIF_DISPLAY_CG_TIME=1 diff --git a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs new file mode 100755 index 00000000000..c70c3ec47f3 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs @@ -0,0 +1,126 @@ +#!/bin/bash +#![forbid(unsafe_code)]/* This line is ignored by bash +# This block is ignored by rustc +CHANNEL="release" +pushd $(dirname "$0")/../ +source scripts/config.sh +popd +PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0 +#*/ + +//! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse +//! profiles. +//! +//! Usage: ./filter_profile.rs <profile in stackcollapse format> <output file> +//! +//! This file is specially crafted to be both a valid bash script and valid rust source file. If +//! executed as bash script this will run the rust source using cg_clif in JIT mode. + +use std::io::Write; + +fn main() -> Result<(), Box<dyn std::error::Error>> { + let profile_name = std::env::var("PROFILE").unwrap(); + let output_name = std::env::var("OUTPUT").unwrap(); + if profile_name.is_empty() || output_name.is_empty() { + println!("Usage: ./filter_profile.rs <profile in stackcollapse format> <output file>"); + std::process::exit(1); + } + let profile = std::fs::read_to_string(profile_name) + .map_err(|err| format!("Failed to read profile {}", err))?; + let mut output = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(output_name)?; + + for line in profile.lines() { + let mut stack = &line[..line.rfind(" ").unwrap()]; + let count = &line[line.rfind(" ").unwrap() + 1..]; + + // Filter away uninteresting samples + if !stack.contains("rustc_codegen_cranelift") { + continue; + } + + if stack.contains("rustc_mir::monomorphize::partitioning::collect_and_partition_mono_items") + || stack.contains("rustc_incremental::assert_dep_graph::assert_dep_graph") + || stack.contains("rustc_symbol_mangling::test::report_symbol_names") + { + continue; + } + + // Trim start + if let Some(index) = stack.find("rustc_interface::passes::configure_and_expand") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::passes::analysis") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::passes::start_codegen") { + stack = &stack[index..]; + } else if let Some(index) = stack.find("rustc_interface::queries::Linker::link") { + stack = &stack[index..]; + } + + if let Some(index) = stack.find("rustc_codegen_cranelift::driver::aot::module_codegen") { + stack = &stack[index..]; + } + + // Trim end + const MALLOC: &str = "malloc"; + if let Some(index) = stack.find(MALLOC) { + stack = &stack[..index + MALLOC.len()]; + } + + const FREE: &str = "free"; + if let Some(index) = stack.find(FREE) { + stack = &stack[..index + FREE.len()]; + } + + const TYPECK_ITEM_BODIES: &str = "rustc_typeck::check::typeck_item_bodies"; + if let Some(index) = stack.find(TYPECK_ITEM_BODIES) { + stack = &stack[..index + TYPECK_ITEM_BODIES.len()]; + } + + const COLLECT_AND_PARTITION_MONO_ITEMS: &str = + "rustc_mir::monomorphize::partitioning::collect_and_partition_mono_items"; + if let Some(index) = stack.find(COLLECT_AND_PARTITION_MONO_ITEMS) { + stack = &stack[..index + COLLECT_AND_PARTITION_MONO_ITEMS.len()]; + } + + const ASSERT_DEP_GRAPH: &str = "rustc_incremental::assert_dep_graph::assert_dep_graph"; + if let Some(index) = stack.find(ASSERT_DEP_GRAPH) { + stack = &stack[..index + ASSERT_DEP_GRAPH.len()]; + } + + const REPORT_SYMBOL_NAMES: &str = "rustc_symbol_mangling::test::report_symbol_names"; + if let Some(index) = stack.find(REPORT_SYMBOL_NAMES) { + stack = &stack[..index + REPORT_SYMBOL_NAMES.len()]; + } + + const ENCODE_METADATA: &str = "rustc_middle::ty::context::TyCtxt::encode_metadata"; + if let Some(index) = stack.find(ENCODE_METADATA) { + stack = &stack[..index + ENCODE_METADATA.len()]; + } + + const SUBST_AND_NORMALIZE_ERASING_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::subst_and_normalize_erasing_regions"; + if let Some(index) = stack.find(SUBST_AND_NORMALIZE_ERASING_REGIONS) { + stack = &stack[..index + SUBST_AND_NORMALIZE_ERASING_REGIONS.len()]; + } + + const NORMALIZE_ERASING_LATE_BOUND_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::normalize_erasing_late_bound_regions"; + if let Some(index) = stack.find(NORMALIZE_ERASING_LATE_BOUND_REGIONS) { + stack = &stack[..index + NORMALIZE_ERASING_LATE_BOUND_REGIONS.len()]; + } + + const INST_BUILD: &str = "<cranelift_frontend::frontend::FuncInstBuilder as cranelift_codegen::ir::builder::InstBuilderBase>::build"; + if let Some(index) = stack.find(INST_BUILD) { + stack = &stack[..index + INST_BUILD.len()]; + } + + output.write_all(stack.as_bytes())?; + output.write_all(&*b" ")?; + output.write_all(count.as_bytes())?; + output.write_all(&*b"\n")?; + } + + Ok(()) +} diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh new file mode 100755 index 00000000000..38991d6d47d --- /dev/null +++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +case $1 in + "prepare") + TOOLCHAIN=$(date +%Y-%m-%d) + + echo "=> Installing new nightly" + rustup toolchain install --profile minimal nightly-${TOOLCHAIN} # Sanity check to see if the nightly exists + echo nightly-${TOOLCHAIN} > rust-toolchain + rustup component add rustfmt || true + + echo "=> Uninstalling all old nighlies" + for nightly in $(rustup toolchain list | grep nightly | grep -v $TOOLCHAIN | grep -v nightly-x86_64); do + rustup toolchain uninstall $nightly + done + + ./clean_all.sh + ./prepare.sh + + (cd build_sysroot && cargo update) + + ;; + "commit") + git add rust-toolchain build_sysroot/Cargo.lock + git commit -m "Rustup to $(rustc -V)" + ;; + *) + echo "Unknown command '$1'" + echo "Usage: ./rustup.sh prepare|commit" + ;; +esac diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs new file mode 100644 index 00000000000..7bb00c8d46a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -0,0 +1,130 @@ +//! Annotate the clif ir with comments describing how arguments are passed into the current function +//! and where all locals are stored. + +use std::borrow::Cow; + +use rustc_middle::mir; + +use cranelift_codegen::entity::EntityRef; + +use crate::abi::pass_mode::*; +use crate::prelude::*; + +pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) { + fx.add_global_comment(format!( + "kind loc.idx param pass mode ty" + )); +} + +pub(super) fn add_arg_comment<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + kind: &str, + local: Option<mir::Local>, + local_field: Option<usize>, + params: EmptySinglePair<Value>, + pass_mode: PassMode, + ty: Ty<'tcx>, +) { + let local = if let Some(local) = local { + Cow::Owned(format!("{:?}", local)) + } else { + Cow::Borrowed("???") + }; + let local_field = if let Some(local_field) = local_field { + Cow::Owned(format!(".{}", local_field)) + } else { + Cow::Borrowed("") + }; + + let params = match params { + Empty => Cow::Borrowed("-"), + Single(param) => Cow::Owned(format!("= {:?}", param)), + Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), + }; + + let pass_mode = format!("{:?}", pass_mode); + fx.add_global_comment(format!( + "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", + kind = kind, + local = local, + local_field = local_field, + params = params, + pass_mode = pass_mode, + ty = ty, + )); +} + +pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) { + fx.add_global_comment(String::new()); + fx.add_global_comment(format!( + "kind local ty size align (abi,pref)" + )); +} + +pub(super) fn add_local_place_comments<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + place: CPlace<'tcx>, + local: Local, +) { + let TyAndLayout { ty, layout } = place.layout(); + let rustc_target::abi::Layout { + size, + align, + abi: _, + variants: _, + fields: _, + largest_niche: _, + } = layout; + + let (kind, extra) = match *place.inner() { + CPlaceInner::Var(place_local, var) => { + assert_eq!(local, place_local); + ("ssa", Cow::Owned(format!(",var={}", var.index()))) + } + CPlaceInner::VarPair(place_local, var1, var2) => { + assert_eq!(local, place_local); + ( + "ssa", + Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())), + ) + } + CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(), + CPlaceInner::Addr(ptr, meta) => { + let meta = if let Some(meta) = meta { + Cow::Owned(format!(",meta={}", meta)) + } else { + Cow::Borrowed("") + }; + match ptr.base_and_offset() { + (crate::pointer::PointerBase::Addr(addr), offset) => ( + "reuse", + format!("storage={}{}{}", addr, offset, meta).into(), + ), + (crate::pointer::PointerBase::Stack(stack_slot), offset) => ( + "stack", + format!("storage={}{}{}", stack_slot, offset, meta).into(), + ), + (crate::pointer::PointerBase::Dangling(align), offset) => ( + "zst", + format!("align={},offset={}", align.bytes(), offset).into(), + ), + } + } + }; + + fx.add_global_comment(format!( + "{:<5} {:5} {:30} {:4}b {}, {}{}{}", + kind, + format!("{:?}", local), + format!("{:?}", ty), + size.bytes(), + align.abi.bytes(), + align.pref.bytes(), + if extra.is_empty() { + "" + } else { + " " + }, + extra, + )); +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs new file mode 100644 index 00000000000..80169122843 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -0,0 +1,766 @@ +//! Handling of everything related to the calling convention. Also fills `fx.local_map`. + +#[cfg(debug_assertions)] +mod comments; +mod pass_mode; +mod returning; + +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_target::spec::abi::Abi; + +use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; + +use self::pass_mode::*; +use crate::prelude::*; + +pub(crate) use self::returning::{can_return_to_ssa_var, codegen_return}; + +// Copied from https://github.com/rust-lang/rust/blob/f52c72948aa1dd718cc1f168d21c91c584c0a662/src/librustc_middle/ty/layout.rs#L2301 +#[rustfmt::skip] +pub(crate) fn fn_sig_for_fn_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> ty::PolyFnSig<'tcx> { + use rustc_middle::ty::subst::Subst; + + // FIXME(davidtwco,eddyb): A `ParamEnv` should be passed through to this function. + let ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); + match *ty.kind() { + ty::FnDef(..) => { + // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering + // parameters unused if they show up in the signature, but not in the `mir::Body` + // (i.e. due to being inside a projection that got normalized, see + // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping + // track of a polymorphization `ParamEnv` to allow normalizing later. + let mut sig = match *ty.kind() { + ty::FnDef(def_id, substs) => tcx + .normalize_erasing_regions(tcx.param_env(def_id), tcx.fn_sig(def_id)) + .subst(tcx, substs), + _ => unreachable!(), + }; + + if let ty::InstanceDef::VtableShim(..) = instance.def { + // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`. + sig = sig.map_bound(|mut sig| { + let mut inputs_and_output = sig.inputs_and_output.to_vec(); + inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]); + sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output); + sig + }); + } + sig + } + ty::Closure(def_id, substs) => { + let sig = substs.as_closure().sig(); + + let env_ty = tcx.closure_env_ty(def_id, substs).unwrap(); + sig.map_bound(|sig| { + tcx.mk_fn_sig( + std::iter::once(env_ty.skip_binder()).chain(sig.inputs().iter().cloned()), + sig.output(), + sig.c_variadic, + sig.unsafety, + sig.abi, + ) + }) + } + ty::Generator(_, substs, _) => { + let sig = substs.as_generator().poly_sig(); + + let env_region = ty::ReLateBound(ty::INNERMOST, ty::BrEnv); + let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty); + + let pin_did = tcx.require_lang_item(rustc_hir::LangItem::Pin, None); + let pin_adt_ref = tcx.adt_def(pin_did); + let pin_substs = tcx.intern_substs(&[env_ty.into()]); + let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs); + + sig.map_bound(|sig| { + let state_did = tcx.require_lang_item(rustc_hir::LangItem::GeneratorState, None); + let state_adt_ref = tcx.adt_def(state_did); + let state_substs = + tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); + let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); + + tcx.mk_fn_sig( + [env_ty, sig.resume_ty].iter(), + &ret_ty, + false, + rustc_hir::Unsafety::Normal, + rustc_target::spec::abi::Abi::Rust, + ) + }) + } + _ => bug!("unexpected type {:?} in Instance::fn_sig", ty), + } +} + +fn clif_sig_from_fn_sig<'tcx>( + tcx: TyCtxt<'tcx>, + triple: &target_lexicon::Triple, + sig: FnSig<'tcx>, + span: Span, + is_vtable_fn: bool, + requires_caller_location: bool, +) -> Signature { + let abi = match sig.abi { + Abi::System => Abi::C, + abi => abi, + }; + let (call_conv, inputs, output): (CallConv, Vec<Ty<'tcx>>, Ty<'tcx>) = match abi { + Abi::Rust => ( + CallConv::triple_default(triple), + sig.inputs().to_vec(), + sig.output(), + ), + Abi::C | Abi::Unadjusted => ( + CallConv::triple_default(triple), + sig.inputs().to_vec(), + sig.output(), + ), + Abi::SysV64 => (CallConv::SystemV, sig.inputs().to_vec(), sig.output()), + Abi::RustCall => { + assert_eq!(sig.inputs().len(), 2); + let extra_args = match sig.inputs().last().unwrap().kind() { + ty::Tuple(ref tupled_arguments) => tupled_arguments, + _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), + }; + let mut inputs: Vec<Ty<'tcx>> = vec![sig.inputs()[0]]; + inputs.extend(extra_args.types()); + (CallConv::triple_default(triple), inputs, sig.output()) + } + Abi::System => unreachable!(), + Abi::RustIntrinsic => ( + CallConv::triple_default(triple), + sig.inputs().to_vec(), + sig.output(), + ), + _ => unimplemented!("unsupported abi {:?}", sig.abi), + }; + + let inputs = inputs + .into_iter() + .enumerate() + .map(|(i, ty)| { + let mut layout = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap(); + if i == 0 && is_vtable_fn { + // Virtual calls turn their self param into a thin pointer. + // See https://github.com/rust-lang/rust/blob/37b6a5e5e82497caf5353d9d856e4eb5d14cbe06/src/librustc/ty/layout.rs#L2519-L2572 for more info + layout = tcx + .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) + .unwrap(); + } + let pass_mode = get_pass_mode(tcx, layout); + if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { + match pass_mode { + PassMode::NoPass | PassMode::ByVal(_) => {} + PassMode::ByRef { size: Some(size) } => { + let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack")); + return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter(); + } + PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => { + tcx.sess.span_warn( + span, + &format!( + "Argument of type `{:?}` with pass mode `{:?}` is not yet supported \ + for non-rust abi `{}`. Calling this function may result in a crash.", + layout.ty, + pass_mode, + abi, + ), + ); + } + } + } + pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter() + }) + .flatten(); + + let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode( + tcx, + tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), + ) { + PassMode::NoPass => (inputs.collect(), vec![]), + PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]), + PassMode::ByValPair(ret_ty_a, ret_ty_b) => ( + inputs.collect(), + vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)], + ), + PassMode::ByRef { size: Some(_) } => { + ( + Some(pointer_ty(tcx)) // First param is place to put return val + .into_iter() + .map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn)) + .chain(inputs) + .collect(), + vec![], + ) + } + PassMode::ByRef { size: None } => todo!(), + }; + + if requires_caller_location { + params.push(AbiParam::new(pointer_ty(tcx))); + } + + Signature { + params, + returns, + call_conv, + } +} + +pub(crate) fn get_function_name_and_sig<'tcx>( + tcx: TyCtxt<'tcx>, + triple: &target_lexicon::Triple, + inst: Instance<'tcx>, + support_vararg: bool, +) -> (String, Signature) { + assert!(!inst.substs.needs_infer()); + let fn_sig = tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + &fn_sig_for_fn_abi(tcx, inst), + ); + if fn_sig.c_variadic && !support_vararg { + tcx.sess.span_fatal( + tcx.def_span(inst.def_id()), + "Variadic function definitions are not yet supported", + ); + } + let sig = clif_sig_from_fn_sig( + tcx, + triple, + fn_sig, + tcx.def_span(inst.def_id()), + false, + inst.def.requires_caller_location(tcx), + ); + (tcx.symbol_name(inst).name.to_string(), sig) +} + +/// Instance must be monomorphized +pub(crate) fn import_function<'tcx>( + tcx: TyCtxt<'tcx>, + module: &mut impl Module, + inst: Instance<'tcx>, +) -> FuncId { + let (name, sig) = get_function_name_and_sig(tcx, module.isa().triple(), inst, true); + module + .declare_function(&name, Linkage::Import, &sig) + .unwrap() +} + +impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { + /// Instance must be monomorphized + pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { + let func_id = import_function(self.tcx, &mut self.cx.module, inst); + let func_ref = self + .cx + .module + .declare_func_in_func(func_id, &mut self.bcx.func); + + #[cfg(debug_assertions)] + self.add_comment(func_ref, format!("{:?}", inst)); + + func_ref + } + + pub(crate) fn lib_call( + &mut self, + name: &str, + input_tys: Vec<types::Type>, + output_tys: Vec<types::Type>, + args: &[Value], + ) -> &[Value] { + let sig = Signature { + params: input_tys.iter().cloned().map(AbiParam::new).collect(), + returns: output_tys.iter().cloned().map(AbiParam::new).collect(), + call_conv: CallConv::triple_default(self.triple()), + }; + let func_id = self + .cx + .module + .declare_function(&name, Linkage::Import, &sig) + .unwrap(); + let func_ref = self + .cx + .module + .declare_func_in_func(func_id, &mut self.bcx.func); + let call_inst = self.bcx.ins().call(func_ref, args); + #[cfg(debug_assertions)] + { + self.add_comment(call_inst, format!("easy_call {}", name)); + } + let results = self.bcx.inst_results(call_inst); + assert!(results.len() <= 2, "{}", results.len()); + results + } + + pub(crate) fn easy_call( + &mut self, + name: &str, + args: &[CValue<'tcx>], + return_ty: Ty<'tcx>, + ) -> CValue<'tcx> { + let (input_tys, args): (Vec<_>, Vec<_>) = args + .into_iter() + .map(|arg| { + ( + self.clif_type(arg.layout().ty).unwrap(), + arg.load_scalar(self), + ) + }) + .unzip(); + let return_layout = self.layout_of(return_ty); + let return_tys = if let ty::Tuple(tup) = return_ty.kind() { + tup.types().map(|ty| self.clif_type(ty).unwrap()).collect() + } else { + vec![self.clif_type(return_ty).unwrap()] + }; + let ret_vals = self.lib_call(name, input_tys, return_tys, &args); + match *ret_vals { + [] => CValue::by_ref( + Pointer::const_addr(self, i64::from(self.pointer_type.bytes())), + return_layout, + ), + [val] => CValue::by_val(val, return_layout), + [val, extra] => CValue::by_val_pair(val, extra, return_layout), + _ => unreachable!(), + } + } +} + +/// Make a [`CPlace`] capable of holding value of the specified type. +fn make_local_place<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + local: Local, + layout: TyAndLayout<'tcx>, + is_ssa: bool, +) -> CPlace<'tcx> { + let place = if is_ssa { + if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi { + CPlace::new_var_pair(fx, local, layout) + } else { + CPlace::new_var(fx, local, layout) + } + } else { + CPlace::new_stack_slot(fx, layout) + }; + + #[cfg(debug_assertions)] + self::comments::add_local_place_comments(fx, place, local); + + place +} + +pub(crate) fn codegen_fn_prelude<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + start_block: Block, +) { + let ssa_analyzed = crate::analyze::analyze(fx); + + #[cfg(debug_assertions)] + self::comments::add_args_header_comment(fx); + + let ret_place = self::returning::codegen_return_param(fx, &ssa_analyzed, start_block); + assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); + + // None means pass_mode == NoPass + enum ArgKind<'tcx> { + Normal(Option<CValue<'tcx>>), + Spread(Vec<Option<CValue<'tcx>>>), + } + + let func_params = fx + .mir + .args_iter() + .map(|local| { + let arg_ty = fx.monomorphize(&fx.mir.local_decls[local].ty); + + // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482 + if Some(local) == fx.mir.spread_arg { + // This argument (e.g. the last argument in the "rust-call" ABI) + // is a tuple that was spread at the ABI level and now we have + // to reconstruct it into a tuple local variable, from multiple + // individual function arguments. + + let tupled_arg_tys = match arg_ty.kind() { + ty::Tuple(ref tys) => tys, + _ => bug!("spread argument isn't a tuple?! but {:?}", arg_ty), + }; + + let mut params = Vec::new(); + for (i, arg_ty) in tupled_arg_tys.types().enumerate() { + let param = cvalue_for_param(fx, start_block, Some(local), Some(i), arg_ty); + params.push(param); + } + + (local, ArgKind::Spread(params), arg_ty) + } else { + let param = cvalue_for_param(fx, start_block, Some(local), None, arg_ty); + (local, ArgKind::Normal(param), arg_ty) + } + }) + .collect::<Vec<(Local, ArgKind<'tcx>, Ty<'tcx>)>>(); + + assert!(fx.caller_location.is_none()); + if fx.instance.def.requires_caller_location(fx.tcx) { + // Store caller location for `#[track_caller]`. + fx.caller_location = Some( + cvalue_for_param(fx, start_block, None, None, fx.tcx.caller_location_ty()).unwrap(), + ); + } + + fx.bcx.switch_to_block(start_block); + fx.bcx.ins().nop(); + + #[cfg(debug_assertions)] + self::comments::add_locals_header_comment(fx); + + for (local, arg_kind, ty) in func_params { + let layout = fx.layout_of(ty); + + let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa; + + // While this is normally an optimization to prevent an unnecessary copy when an argument is + // not mutated by the current function, this is necessary to support unsized arguments. + match arg_kind { + ArgKind::Normal(Some(val)) => { + if let Some((addr, meta)) = val.try_to_ptr() { + let local_decl = &fx.mir.local_decls[local]; + // v this ! is important + let internally_mutable = !val.layout().ty.is_freeze( + fx.tcx.at(local_decl.source_info.span), + ParamEnv::reveal_all(), + ); + if local_decl.mutability == mir::Mutability::Not && !internally_mutable { + // We wont mutate this argument, so it is fine to borrow the backing storage + // of this argument, to prevent a copy. + + let place = if let Some(meta) = meta { + CPlace::for_ptr_with_extra(addr, meta, val.layout()) + } else { + CPlace::for_ptr(addr, val.layout()) + }; + + #[cfg(debug_assertions)] + self::comments::add_local_place_comments(fx, place, local); + + assert_eq!(fx.local_map.push(place), local); + continue; + } + } + } + _ => {} + } + + let place = make_local_place(fx, local, layout, is_ssa); + assert_eq!(fx.local_map.push(place), local); + + match arg_kind { + ArgKind::Normal(param) => { + if let Some(param) = param { + place.write_cvalue(fx, param); + } + } + ArgKind::Spread(params) => { + for (i, param) in params.into_iter().enumerate() { + if let Some(param) = param { + place + .place_field(fx, mir::Field::new(i)) + .write_cvalue(fx, param); + } + } + } + } + } + + for local in fx.mir.vars_and_temps_iter() { + let ty = fx.monomorphize(&fx.mir.local_decls[local].ty); + let layout = fx.layout_of(ty); + + let is_ssa = ssa_analyzed[local] == crate::analyze::SsaKind::Ssa; + + let place = make_local_place(fx, local, layout, is_ssa); + assert_eq!(fx.local_map.push(place), local); + } + + fx.bcx + .ins() + .jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]); +} + +pub(crate) fn codegen_terminator_call<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + span: Span, + current_block: Block, + func: &Operand<'tcx>, + args: &[Operand<'tcx>], + destination: Option<(Place<'tcx>, BasicBlock)>, +) { + let fn_ty = fx.monomorphize(&func.ty(fx.mir, fx.tcx)); + let fn_sig = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), &fn_ty.fn_sig(fx.tcx)); + + let destination = destination.map(|(place, bb)| (trans_place(fx, place), bb)); + + // Handle special calls like instrinsics and empty drop glue. + let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() { + let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs) + .unwrap() + .unwrap() + .polymorphize(fx.tcx); + + if fx.tcx.symbol_name(instance).name.starts_with("llvm.") { + crate::intrinsics::codegen_llvm_intrinsic_call( + fx, + &fx.tcx.symbol_name(instance).name, + substs, + args, + destination, + ); + return; + } + + match instance.def { + InstanceDef::Intrinsic(_) => { + crate::intrinsics::codegen_intrinsic_call(fx, instance, args, destination, span); + return; + } + InstanceDef::DropGlue(_, None) => { + // empty drop glue - a nop. + let (_, dest) = destination.expect("Non terminating drop_in_place_real???"); + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + return; + } + _ => Some(instance), + } + } else { + None + }; + + let is_cold = instance + .map(|inst| { + fx.tcx + .codegen_fn_attrs(inst.def_id()) + .flags + .contains(CodegenFnAttrFlags::COLD) + }) + .unwrap_or(false); + if is_cold { + fx.cold_blocks.insert(current_block); + } + + // Unpack arguments tuple for closures + let args = if fn_sig.abi == Abi::RustCall { + assert_eq!(args.len(), 2, "rust-call abi requires two arguments"); + let self_arg = trans_operand(fx, &args[0]); + let pack_arg = trans_operand(fx, &args[1]); + + let tupled_arguments = match pack_arg.layout().ty.kind() { + ty::Tuple(ref tupled_arguments) => tupled_arguments, + _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), + }; + + let mut args = Vec::with_capacity(1 + tupled_arguments.len()); + args.push(self_arg); + for i in 0..tupled_arguments.len() { + args.push(pack_arg.value_field(fx, mir::Field::new(i))); + } + args + } else { + args.into_iter() + .map(|arg| trans_operand(fx, arg)) + .collect::<Vec<_>>() + }; + + // | indirect call target + // | | the first argument to be passed + // v v v virtual calls are special cased below + let (func_ref, first_arg, is_virtual_call) = match instance { + // Trait object call + Some(Instance { + def: InstanceDef::Virtual(_, idx), + .. + }) => { + #[cfg(debug_assertions)] + { + let nop_inst = fx.bcx.ins().nop(); + fx.add_comment( + nop_inst, + format!( + "virtual call; self arg pass mode: {:?}", + get_pass_mode(fx.tcx, args[0].layout()) + ), + ); + } + let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0], idx); + (Some(method), Single(ptr), true) + } + + // Normal call + Some(_) => ( + None, + args.get(0) + .map(|arg| adjust_arg_for_abi(fx, *arg)) + .unwrap_or(Empty), + false, + ), + + // Indirect call + None => { + #[cfg(debug_assertions)] + { + let nop_inst = fx.bcx.ins().nop(); + fx.add_comment(nop_inst, "indirect call"); + } + let func = trans_operand(fx, func).load_scalar(fx); + ( + Some(func), + args.get(0) + .map(|arg| adjust_arg_for_abi(fx, *arg)) + .unwrap_or(Empty), + false, + ) + } + }; + + let ret_place = destination.map(|(place, _)| place); + let (call_inst, call_args) = + self::returning::codegen_with_call_return_arg(fx, fn_sig, ret_place, |fx, return_ptr| { + let mut call_args: Vec<Value> = return_ptr + .into_iter() + .chain(first_arg.into_iter()) + .chain( + args.into_iter() + .skip(1) + .map(|arg| adjust_arg_for_abi(fx, arg).into_iter()) + .flatten(), + ) + .collect::<Vec<_>>(); + + if instance + .map(|inst| inst.def.requires_caller_location(fx.tcx)) + .unwrap_or(false) + { + // Pass the caller location for `#[track_caller]`. + let caller_location = fx.get_caller_location(span); + call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + } + + let call_inst = if let Some(func_ref) = func_ref { + let sig = clif_sig_from_fn_sig( + fx.tcx, + fx.triple(), + fn_sig, + span, + is_virtual_call, + false, // calls through function pointers never pass the caller location + ); + let sig = fx.bcx.import_signature(sig); + fx.bcx.ins().call_indirect(sig, func_ref, &call_args) + } else { + let func_ref = + fx.get_function_ref(instance.expect("non-indirect call on non-FnDef type")); + fx.bcx.ins().call(func_ref, &call_args) + }; + + (call_inst, call_args) + }); + + // FIXME find a cleaner way to support varargs + if fn_sig.c_variadic { + if fn_sig.abi != Abi::C { + fx.tcx.sess.span_fatal( + span, + &format!("Variadic call for non-C abi {:?}", fn_sig.abi), + ); + } + let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap(); + let abi_params = call_args + .into_iter() + .map(|arg| { + let ty = fx.bcx.func.dfg.value_type(arg); + if !ty.is_int() { + // FIXME set %al to upperbound on float args once floats are supported + fx.tcx + .sess + .span_fatal(span, &format!("Non int ty {:?} for variadic call", ty)); + } + AbiParam::new(ty) + }) + .collect::<Vec<AbiParam>>(); + fx.bcx.func.dfg.signatures[sig_ref].params = abi_params; + } + + if let Some((_, dest)) = destination { + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + } else { + trap_unreachable(fx, "[corruption] Diverging function returned"); + } +} + +pub(crate) fn codegen_drop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + span: Span, + drop_place: CPlace<'tcx>, +) { + let ty = drop_place.layout().ty; + let drop_fn = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); + + if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def { + // we don't actually need to drop anything + } else { + let drop_fn_ty = drop_fn.ty(fx.tcx, ParamEnv::reveal_all()); + let fn_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + &drop_fn_ty.fn_sig(fx.tcx), + ); + assert_eq!(fn_sig.output(), fx.tcx.mk_unit()); + + match ty.kind() { + ty::Dynamic(..) => { + let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); + let ptr = ptr.get_addr(fx); + let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); + + let sig = clif_sig_from_fn_sig( + fx.tcx, + fx.triple(), + fn_sig, + span, + true, + false, // `drop_in_place` is never `#[track_caller]` + ); + let sig = fx.bcx.import_signature(sig); + fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); + } + _ => { + assert!(!matches!(drop_fn.def, InstanceDef::Virtual(_, _))); + + let arg_value = drop_place.place_ref( + fx, + fx.layout_of(fx.tcx.mk_ref( + &ty::RegionKind::ReErased, + TypeAndMut { + ty, + mutbl: crate::rustc_hir::Mutability::Mut, + }, + )), + ); + let arg_value = adjust_arg_for_abi(fx, arg_value); + + let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>(); + + if drop_fn.def.requires_caller_location(fx.tcx) { + // Pass the caller location for `#[track_caller]`. + let caller_location = fx.get_caller_location(span); + call_args.extend(adjust_arg_for_abi(fx, caller_location).into_iter()); + } + + let func_ref = fx.get_function_ref(drop_fn); + fx.bcx.ins().call(func_ref, &call_args); + } + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs new file mode 100644 index 00000000000..8e3682c86c5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -0,0 +1,188 @@ +//! Argument passing + +use crate::prelude::*; + +pub(super) use EmptySinglePair::*; + +#[derive(Copy, Clone, Debug)] +pub(super) enum PassMode { + NoPass, + ByVal(Type), + ByValPair(Type, Type), + ByRef { size: Option<Size> }, +} + +#[derive(Copy, Clone, Debug)] +pub(super) enum EmptySinglePair<T> { + Empty, + Single(T), + Pair(T, T), +} + +impl<T> EmptySinglePair<T> { + pub(super) fn into_iter(self) -> EmptySinglePairIter<T> { + EmptySinglePairIter(self) + } + + pub(super) fn map<U>(self, mut f: impl FnMut(T) -> U) -> EmptySinglePair<U> { + match self { + Empty => Empty, + Single(v) => Single(f(v)), + Pair(a, b) => Pair(f(a), f(b)), + } + } +} + +pub(super) struct EmptySinglePairIter<T>(EmptySinglePair<T>); + +impl<T> Iterator for EmptySinglePairIter<T> { + type Item = T; + + fn next(&mut self) -> Option<T> { + match std::mem::replace(&mut self.0, Empty) { + Empty => None, + Single(v) => Some(v), + Pair(a, b) => { + self.0 = Single(b); + Some(a) + } + } + } +} + +impl<T: std::fmt::Debug> EmptySinglePair<T> { + pub(super) fn assert_single(self) -> T { + match self { + Single(v) => v, + _ => panic!("Called assert_single on {:?}", self), + } + } + + pub(super) fn assert_pair(self) -> (T, T) { + match self { + Pair(a, b) => (a, b), + _ => panic!("Called assert_pair on {:?}", self), + } + } +} + +impl PassMode { + pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair<Type> { + match self { + PassMode::NoPass => Empty, + PassMode::ByVal(clif_type) => Single(clif_type), + PassMode::ByValPair(a, b) => Pair(a, b), + PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)), + PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)), + } + } +} + +pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { + if layout.is_zst() { + // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer + PassMode::NoPass + } else { + match &layout.abi { + Abi::Uninhabited => PassMode::NoPass, + Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + if a == types::I128 && b == types::I128 { + // Returning (i128, i128) by-val-pair would take 4 regs, while only 3 are + // available on x86_64. Cranelift gets confused when too many return params + // are used. + PassMode::ByRef { + size: Some(layout.size), + } + } else { + PassMode::ByValPair(a, b) + } + } + + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + if let Some(vector_ty) = crate::intrinsics::clif_vector_type(tcx, layout) { + PassMode::ByVal(vector_ty) + } else { + PassMode::ByRef { + size: Some(layout.size), + } + } + } + + Abi::Aggregate { sized: true } => PassMode::ByRef { + size: Some(layout.size), + }, + Abi::Aggregate { sized: false } => PassMode::ByRef { size: None }, + } + } +} + +/// Get a set of values to be passed as function arguments. +pub(super) fn adjust_arg_for_abi<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + arg: CValue<'tcx>, +) -> EmptySinglePair<Value> { + match get_pass_mode(fx.tcx, arg.layout()) { + PassMode::NoPass => Empty, + PassMode::ByVal(_) => Single(arg.load_scalar(fx)), + PassMode::ByValPair(_, _) => { + let (a, b) = arg.load_scalar_pair(fx); + Pair(a, b) + } + PassMode::ByRef { size: _ } => match arg.force_stack(fx) { + (ptr, None) => Single(ptr.get_addr(fx)), + (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), + }, + } +} + +/// Create a [`CValue`] containing the value of a function parameter adding clif function parameters +/// as necessary. +pub(super) fn cvalue_for_param<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + start_block: Block, + #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>, + #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>, + arg_ty: Ty<'tcx>, +) -> Option<CValue<'tcx>> { + let layout = fx.layout_of(arg_ty); + let pass_mode = get_pass_mode(fx.tcx, layout); + + if let PassMode::NoPass = pass_mode { + return None; + } + + let clif_types = pass_mode.get_param_ty(fx.tcx); + let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t)); + + #[cfg(debug_assertions)] + crate::abi::comments::add_arg_comment( + fx, + "arg", + local, + local_field, + block_params, + pass_mode, + arg_ty, + ); + + match pass_mode { + PassMode::NoPass => unreachable!(), + PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)), + PassMode::ByValPair(_, _) => { + let (a, b) = block_params.assert_pair(); + Some(CValue::by_val_pair(a, b, layout)) + } + PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref( + Pointer::new(block_params.assert_single()), + layout, + )), + PassMode::ByRef { size: None } => { + let (ptr, meta) = block_params.assert_pair(); + Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs new file mode 100644 index 00000000000..f6d40c880d0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -0,0 +1,130 @@ +//! Return value handling + +use crate::abi::pass_mode::*; +use crate::prelude::*; + +fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { + fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) +} + +/// Can the given type be returned into an ssa var or does it need to be returned on the stack. +pub(crate) fn can_return_to_ssa_var<'tcx>( + tcx: TyCtxt<'tcx>, + dest_layout: TyAndLayout<'tcx>, +) -> bool { + match get_pass_mode(tcx, dest_layout) { + PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true, + // FIXME Make it possible to return ByRef to an ssa var. + PassMode::ByRef { size: _ } => false, + } +} + +/// Return a place where the return value of the current function can be written to. If necessary +/// this adds an extra parameter pointing to where the return value needs to be stored. +pub(super) fn codegen_return_param<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>, + start_block: Block, +) -> CPlace<'tcx> { + let ret_layout = return_layout(fx); + let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout); + let (ret_place, ret_param) = match ret_pass_mode { + PassMode::NoPass => (CPlace::no_place(ret_layout), Empty), + PassMode::ByVal(_) | PassMode::ByValPair(_, _) => { + let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; + ( + super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), + Empty, + ) + } + PassMode::ByRef { size: Some(_) } => { + let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); + ( + CPlace::for_ptr(Pointer::new(ret_param), ret_layout), + Single(ret_param), + ) + } + PassMode::ByRef { size: None } => todo!(), + }; + + #[cfg(not(debug_assertions))] + let _ = ret_param; + + #[cfg(debug_assertions)] + crate::abi::comments::add_arg_comment( + fx, + "ret", + Some(RETURN_PLACE), + None, + ret_param, + ret_pass_mode, + ret_layout.ty, + ); + + ret_place +} + +/// Invokes the closure with if necessary a value representing the return pointer. When the closure +/// returns the call return value(s) if any are written to the correct place. +pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( + fx: &mut FunctionCx<'_, 'tcx, M>, + fn_sig: FnSig<'tcx>, + ret_place: Option<CPlace<'tcx>>, + f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option<Value>) -> (Inst, T), +) -> (Inst, T) { + let ret_layout = fx.layout_of(fn_sig.output()); + + let output_pass_mode = get_pass_mode(fx.tcx, ret_layout); + let return_ptr = match output_pass_mode { + PassMode::NoPass => None, + PassMode::ByRef { size: Some(_) } => match ret_place { + Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), + None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot + }, + PassMode::ByRef { size: None } => todo!(), + PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None, + }; + + let (call_inst, meta) = f(fx, return_ptr); + + match output_pass_mode { + PassMode::NoPass => {} + PassMode::ByVal(_) => { + if let Some(ret_place) = ret_place { + let ret_val = fx.bcx.inst_results(call_inst)[0]; + ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); + } + } + PassMode::ByValPair(_, _) => { + if let Some(ret_place) = ret_place { + let ret_val_a = fx.bcx.inst_results(call_inst)[0]; + let ret_val_b = fx.bcx.inst_results(call_inst)[1]; + ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); + } + } + PassMode::ByRef { size: Some(_) } => {} + PassMode::ByRef { size: None } => todo!(), + } + + (call_inst, meta) +} + +/// Codegen a return instruction with the right return value(s) if any. +pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { + match get_pass_mode(fx.tcx, return_layout(fx)) { + PassMode::NoPass | PassMode::ByRef { size: Some(_) } => { + fx.bcx.ins().return_(&[]); + } + PassMode::ByRef { size: None } => todo!(), + PassMode::ByVal(_) => { + let place = fx.get_local_place(RETURN_PLACE); + let ret_val = place.to_cvalue(fx).load_scalar(fx); + fx.bcx.ins().return_(&[ret_val]); + } + PassMode::ByValPair(_, _) => { + let place = fx.get_local_place(RETURN_PLACE); + let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); + fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs new file mode 100644 index 00000000000..0735ad6f832 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -0,0 +1,153 @@ +//! Allocator shim +// Adapted from rustc + +use crate::prelude::*; + +use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_span::symbol::sym; + +/// Returns whether an allocator shim was created +pub(crate) fn codegen( + tcx: TyCtxt<'_>, + module: &mut impl Module, + unwind_context: &mut UnwindContext<'_>, +) -> bool { + let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { + use rustc_middle::middle::dependency_format::Linkage; + list.iter().any(|&linkage| linkage == Linkage::Dynamic) + }); + if any_dynamic_crate { + false + } else if let Some(kind) = tcx.allocator_kind() { + codegen_inner(module, unwind_context, kind); + true + } else { + false + } +} + +fn codegen_inner( + module: &mut impl Module, + unwind_context: &mut UnwindContext<'_>, + kind: AllocatorKind, +) { + let usize_ty = module.target_config().pointer_type(); + + for method in ALLOCATOR_METHODS { + let mut arg_tys = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + arg_tys.push(usize_ty); // size + arg_tys.push(usize_ty); // align + } + AllocatorTy::Ptr => arg_tys.push(usize_ty), + AllocatorTy::Usize => arg_tys.push(usize_ty), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } + } + let output = match method.output { + AllocatorTy::ResultPtr => Some(usize_ty), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + + let sig = Signature { + call_conv: CallConv::triple_default(module.isa().triple()), + params: arg_tys.iter().cloned().map(AbiParam::new).collect(), + returns: output.into_iter().map(AbiParam::new).collect(), + }; + + let caller_name = format!("__rust_{}", method.name); + let callee_name = kind.fn_name(method.name); + //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns); + + let func_id = module + .declare_function(&caller_name, Linkage::Export, &sig) + .unwrap(); + + let callee_func_id = module + .declare_function(&callee_name, Linkage::Import, &sig) + .unwrap(); + + let mut ctx = Context::new(); + ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone()); + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let args = arg_tys + .into_iter() + .map(|ty| bcx.append_block_param(block, ty)) + .collect::<Vec<Value>>(); + + let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); + let call_inst = bcx.ins().call(callee_func_ref, &args); + let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error + + bcx.ins().return_(&results); + bcx.seal_all_blocks(); + bcx.finalize(); + } + module + .define_function( + func_id, + &mut ctx, + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); + unwind_context.add_function(func_id, &ctx, module.isa()); + } + + let sig = Signature { + call_conv: CallConv::triple_default(module.isa().triple()), + params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)], + returns: vec![], + }; + + let callee_name = kind.fn_name(sym::oom); + //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns); + + let func_id = module + .declare_function("__rust_alloc_error_handler", Linkage::Export, &sig) + .unwrap(); + + let callee_func_id = module + .declare_function(&callee_name, Linkage::Import, &sig) + .unwrap(); + + let mut ctx = Context::new(); + ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone()); + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let args = (&[usize_ty, usize_ty]) + .into_iter() + .map(|&ty| bcx.append_block_param(block, ty)) + .collect::<Vec<Value>>(); + + let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); + bcx.ins().call(callee_func_ref, &args); + + bcx.ins().trap(TrapCode::UnreachableCodeReached); + bcx.seal_all_blocks(); + bcx.finalize(); + } + module + .define_function( + func_id, + &mut ctx, + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); + unwind_context.add_function(func_id, &ctx, module.isa()); +} diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs new file mode 100644 index 00000000000..fd25b19a583 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/analyze.rs @@ -0,0 +1,61 @@ +//! SSA analysis + +use crate::prelude::*; + +use rustc_index::vec::IndexVec; +use rustc_middle::mir::StatementKind::*; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum SsaKind { + NotSsa, + Ssa, +} + +pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, SsaKind> { + let mut flag_map = fx + .mir + .local_decls + .iter() + .map(|local_decl| { + let ty = fx.monomorphize(&local_decl.ty); + if fx.clif_type(ty).is_some() || fx.clif_pair_type(ty).is_some() { + SsaKind::Ssa + } else { + SsaKind::NotSsa + } + }) + .collect::<IndexVec<Local, SsaKind>>(); + + for bb in fx.mir.basic_blocks().iter() { + for stmt in bb.statements.iter() { + match &stmt.kind { + Assign(place_and_rval) => match &place_and_rval.1 { + Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { + not_ssa(&mut flag_map, place.local) + } + _ => {} + }, + _ => {} + } + } + + match &bb.terminator().kind { + TerminatorKind::Call { destination, .. } => { + if let Some((dest_place, _dest_bb)) = destination { + let dest_layout = fx + .layout_of(fx.monomorphize(&dest_place.ty(&fx.mir.local_decls, fx.tcx).ty)); + if !crate::abi::can_return_to_ssa_var(fx.tcx, dest_layout) { + not_ssa(&mut flag_map, dest_place.local) + } + } + } + _ => {} + } + } + + flag_map +} + +fn not_ssa(flag_map: &mut IndexVec<Local, SsaKind>, local: Local) { + flag_map[local] = SsaKind::NotSsa; +} diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs new file mode 100644 index 00000000000..6382f8df344 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -0,0 +1,309 @@ +//! Creation of ar archives like for the lib and staticlib crate type + +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Path, PathBuf}; + +use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder}; +use rustc_codegen_ssa::METADATA_FILENAME; +use rustc_session::Session; + +use object::{Object, SymbolKind}; + +#[derive(Debug)] +enum ArchiveEntry { + FromArchive { + archive_index: usize, + entry_index: usize, + }, + File(PathBuf), +} + +pub(crate) struct ArArchiveBuilder<'a> { + sess: &'a Session, + dst: PathBuf, + lib_search_paths: Vec<PathBuf>, + use_gnu_style_archive: bool, + no_builtin_ranlib: bool, + + src_archives: Vec<(PathBuf, ar::Archive<File>)>, + // Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at + // the end of an archive for linkers to not get confused. + entries: Vec<(String, ArchiveEntry)>, + update_symbols: bool, +} + +impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { + fn new(sess: &'a Session, output: &Path, input: Option<&Path>) -> Self { + use rustc_codegen_ssa::back::link::archive_search_paths; + + let (src_archives, entries) = if let Some(input) = input { + let mut archive = ar::Archive::new(File::open(input).unwrap()); + let mut entries = Vec::new(); + + let mut i = 0; + while let Some(entry) = archive.next_entry() { + let entry = entry.unwrap(); + entries.push(( + String::from_utf8(entry.header().identifier().to_vec()).unwrap(), + ArchiveEntry::FromArchive { + archive_index: 0, + entry_index: i, + }, + )); + i += 1; + } + + (vec![(input.to_owned(), archive)], entries) + } else { + (vec![], Vec::new()) + }; + + ArArchiveBuilder { + sess, + dst: output.to_path_buf(), + lib_search_paths: archive_search_paths(sess), + use_gnu_style_archive: sess.target.options.archive_format == "gnu", + // FIXME fix builtin ranlib on macOS + no_builtin_ranlib: sess.target.options.is_like_osx, + + src_archives, + entries, + update_symbols: false, + } + } + + fn src_files(&mut self) -> Vec<String> { + self.entries.iter().map(|(name, _)| name.clone()).collect() + } + + fn remove_file(&mut self, name: &str) { + let index = self + .entries + .iter() + .position(|(entry_name, _)| entry_name == name) + .expect("Tried to remove file not existing in src archive"); + self.entries.remove(index); + } + + fn add_file(&mut self, file: &Path) { + self.entries.push(( + file.file_name().unwrap().to_str().unwrap().to_string(), + ArchiveEntry::File(file.to_owned()), + )); + } + + fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) { + let location = find_library(name, &self.lib_search_paths, self.sess); + self.add_archive(location.clone(), |_| false) + .unwrap_or_else(|e| { + panic!( + "failed to add native library {}: {}", + location.to_string_lossy(), + e + ); + }); + } + + fn add_rlib( + &mut self, + rlib: &Path, + name: &str, + lto: bool, + skip_objects: bool, + ) -> std::io::Result<()> { + let obj_start = name.to_owned(); + + self.add_archive(rlib.to_owned(), move |fname: &str| { + // Ignore metadata files, no matter the name. + if fname == METADATA_FILENAME { + return true; + } + + // Don't include Rust objects if LTO is enabled + if lto && fname.starts_with(&obj_start) && fname.ends_with(".o") { + return true; + } + + // Otherwise if this is *not* a rust object and we're skipping + // objects then skip this file + if skip_objects && (!fname.starts_with(&obj_start) || !fname.ends_with(".o")) { + return true; + } + + // ok, don't skip this + return false; + }) + } + + fn update_symbols(&mut self) { + self.update_symbols = true; + } + + fn build(mut self) { + enum BuilderKind { + Bsd(ar::Builder<File>), + Gnu(ar::GnuBuilder<File>), + } + + let sess = self.sess; + + let mut symbol_table = BTreeMap::new(); + + let mut entries = Vec::new(); + + for (entry_name, entry) in self.entries { + // FIXME only read the symbol table of the object files to avoid having to keep all + // object files in memory at once, or read them twice. + let data = match entry { + ArchiveEntry::FromArchive { + archive_index, + entry_index, + } => { + // FIXME read symbols from symtab + use std::io::Read; + let (ref _src_archive_path, ref mut src_archive) = + self.src_archives[archive_index]; + let mut entry = src_archive.jump_to_entry(entry_index).unwrap(); + let mut data = Vec::new(); + entry.read_to_end(&mut data).unwrap(); + data + } + ArchiveEntry::File(file) => std::fs::read(file).unwrap_or_else(|err| { + sess.fatal(&format!( + "error while reading object file during archive building: {}", + err + )); + }), + }; + + if !self.no_builtin_ranlib { + match object::File::parse(&data) { + Ok(object) => { + symbol_table.insert( + entry_name.as_bytes().to_vec(), + object + .symbols() + .filter_map(|(_index, symbol)| { + if symbol.is_undefined() + || symbol.is_local() + || symbol.kind() != SymbolKind::Data + && symbol.kind() != SymbolKind::Text + && symbol.kind() != SymbolKind::Tls + { + None + } else { + symbol.name().map(|name| name.as_bytes().to_vec()) + } + }) + .collect::<Vec<_>>(), + ); + } + Err(err) => { + let err = err.to_string(); + if err == "Unknown file magic" { + // Not an object file; skip it. + } else { + sess.fatal(&format!( + "error parsing `{}` during archive creation: {}", + entry_name, err + )); + } + } + } + } + + entries.push((entry_name, data)); + } + + let mut builder = if self.use_gnu_style_archive { + BuilderKind::Gnu( + ar::GnuBuilder::new( + File::create(&self.dst).unwrap_or_else(|err| { + sess.fatal(&format!( + "error opening destination during archive building: {}", + err + )); + }), + entries + .iter() + .map(|(name, _)| name.as_bytes().to_vec()) + .collect(), + ar::GnuSymbolTableFormat::Size32, + symbol_table, + ) + .unwrap(), + ) + } else { + BuilderKind::Bsd( + ar::Builder::new( + File::create(&self.dst).unwrap_or_else(|err| { + sess.fatal(&format!( + "error opening destination during archive building: {}", + err + )); + }), + symbol_table, + ) + .unwrap(), + ) + }; + + // Add all files + for (entry_name, data) in entries.into_iter() { + let header = ar::Header::new(entry_name.into_bytes(), data.len() as u64); + match builder { + BuilderKind::Bsd(ref mut builder) => builder.append(&header, &mut &*data).unwrap(), + BuilderKind::Gnu(ref mut builder) => builder.append(&header, &mut &*data).unwrap(), + } + } + + // Finalize archive + std::mem::drop(builder); + + if self.no_builtin_ranlib { + let ranlib = crate::toolchain::get_toolchain_binary(self.sess, "ranlib"); + + // Run ranlib to be able to link the archive + let status = std::process::Command::new(ranlib) + .arg(self.dst) + .status() + .expect("Couldn't run ranlib"); + + if !status.success() { + self.sess + .fatal(&format!("Ranlib exited with code {:?}", status.code())); + } + } + } +} + +impl<'a> ArArchiveBuilder<'a> { + fn add_archive<F>(&mut self, archive_path: PathBuf, mut skip: F) -> std::io::Result<()> + where + F: FnMut(&str) -> bool + 'static, + { + let mut archive = ar::Archive::new(std::fs::File::open(&archive_path)?); + let archive_index = self.src_archives.len(); + + let mut i = 0; + while let Some(entry) = archive.next_entry() { + let entry = entry?; + let file_name = String::from_utf8(entry.header().identifier().to_vec()) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + if !skip(&file_name) { + self.entries.push(( + file_name, + ArchiveEntry::FromArchive { + archive_index, + entry_index: i, + }, + )); + } + i += 1; + } + + self.src_archives.push((archive_path, archive)); + Ok(()) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/atomic_shim.rs b/compiler/rustc_codegen_cranelift/src/atomic_shim.rs new file mode 100644 index 00000000000..92281fdacc9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/atomic_shim.rs @@ -0,0 +1,186 @@ +//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support +//! atomic operations yet. + +// FIXME implement atomic instructions in Cranelift. + +use crate::prelude::*; + +#[cfg(all(feature = "jit", unix))] +#[no_mangle] +pub static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = + libc::PTHREAD_MUTEX_INITIALIZER; + +pub(crate) fn init_global_lock( + module: &mut impl Module, + bcx: &mut FunctionBuilder<'_>, + use_jit: bool, +) { + if use_jit { + // When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here, + // so instead we define it in the cg_clif dylib. + + return; + } + + let mut data_ctx = DataContext::new(); + data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms. + data_ctx.set_align(16); + let atomic_mutex = module + .declare_data( + "__cg_clif_global_atomic_mutex", + Linkage::Export, + true, + false, + ) + .unwrap(); + module.define_data(atomic_mutex, &data_ctx).unwrap(); + + let pthread_mutex_init = module + .declare_function( + "pthread_mutex_init", + Linkage::Import, + &cranelift_codegen::ir::Signature { + call_conv: module.target_config().default_call_conv, + params: vec![ + AbiParam::new( + module.target_config().pointer_type(), /* *mut pthread_mutex_t */ + ), + AbiParam::new( + module.target_config().pointer_type(), /* *const pthread_mutex_attr_t */ + ), + ], + returns: vec![AbiParam::new(types::I32 /* c_int */)], + }, + ) + .unwrap(); + + let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func); + + let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func); + let atomic_mutex = bcx + .ins() + .global_value(module.target_config().pointer_type(), atomic_mutex); + + let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0); + + bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]); +} + +pub(crate) fn init_global_lock_constructor( + module: &mut impl Module, + constructor_name: &str, +) -> FuncId { + let sig = Signature::new(CallConv::SystemV); + let init_func_id = module + .declare_function(constructor_name, Linkage::Export, &sig) + .unwrap(); + + let mut ctx = Context::new(); + ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig); + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + + crate::atomic_shim::init_global_lock(module, &mut bcx, false); + + bcx.ins().return_(&[]); + bcx.seal_all_blocks(); + bcx.finalize(); + } + module + .define_function( + init_func_id, + &mut ctx, + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); + + init_func_id +} + +pub(crate) fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) { + let atomic_mutex = fx + .cx + .module + .declare_data( + "__cg_clif_global_atomic_mutex", + Linkage::Import, + true, + false, + ) + .unwrap(); + + let pthread_mutex_lock = fx + .cx + .module + .declare_function( + "pthread_mutex_lock", + Linkage::Import, + &cranelift_codegen::ir::Signature { + call_conv: fx.cx.module.target_config().default_call_conv, + params: vec![AbiParam::new( + fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */ + )], + returns: vec![AbiParam::new(types::I32 /* c_int */)], + }, + ) + .unwrap(); + + let pthread_mutex_lock = fx + .cx + .module + .declare_func_in_func(pthread_mutex_lock, fx.bcx.func); + + let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func); + let atomic_mutex = fx + .bcx + .ins() + .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex); + + fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]); +} + +pub(crate) fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) { + let atomic_mutex = fx + .cx + .module + .declare_data( + "__cg_clif_global_atomic_mutex", + Linkage::Import, + true, + false, + ) + .unwrap(); + + let pthread_mutex_unlock = fx + .cx + .module + .declare_function( + "pthread_mutex_unlock", + Linkage::Import, + &cranelift_codegen::ir::Signature { + call_conv: fx.cx.module.target_config().default_call_conv, + params: vec![AbiParam::new( + fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */ + )], + returns: vec![AbiParam::new(types::I32 /* c_int */)], + }, + ) + .unwrap(); + + let pthread_mutex_unlock = fx + .cx + .module + .declare_func_in_func(pthread_mutex_unlock, fx.bcx.func); + + let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func); + let atomic_mutex = fx + .bcx + .ins() + .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex); + + fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]); +} diff --git a/compiler/rustc_codegen_cranelift/src/backend.rs b/compiler/rustc_codegen_cranelift/src/backend.rs new file mode 100644 index 00000000000..8b900fd0dd0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/backend.rs @@ -0,0 +1,206 @@ +//! Abstraction around the object writing crate + +use std::convert::{TryFrom, TryInto}; + +use rustc_data_structures::fx::FxHashMap; +use rustc_session::Session; + +use cranelift_module::FuncId; + +use object::write::*; +use object::{RelocationEncoding, RelocationKind, SectionKind, SymbolFlags}; + +use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; + +use gimli::SectionId; + +use crate::debuginfo::{DebugReloc, DebugRelocName}; + +pub(crate) trait WriteMetadata { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, is_like_osx: bool); +} + +impl WriteMetadata for object::write::Object { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) { + let segment = self + .segment_name(object::write::StandardSegment::Data) + .to_vec(); + let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); + let offset = self.append_section_data(section_id, &data, 1); + // For MachO and probably PE this is necessary to prevent the linker from throwing away the + // .rustc section. For ELF this isn't necessary, but it also doesn't harm. + self.add_symbol(object::write::Symbol { + name: symbol_name.into_bytes(), + value: offset, + size: data.len() as u64, + kind: object::SymbolKind::Data, + scope: object::SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(section_id), + flags: SymbolFlags::None, + }); + } +} + +pub(crate) trait WriteDebugInfo { + type SectionId: Copy; + + fn add_debug_section(&mut self, name: SectionId, data: Vec<u8>) -> Self::SectionId; + fn add_debug_reloc( + &mut self, + section_map: &FxHashMap<SectionId, Self::SectionId>, + from: &Self::SectionId, + reloc: &DebugReloc, + ); +} + +impl WriteDebugInfo for ObjectProduct { + type SectionId = (object::write::SectionId, object::write::SymbolId); + + fn add_debug_section( + &mut self, + id: SectionId, + data: Vec<u8>, + ) -> (object::write::SectionId, object::write::SymbolId) { + let name = if self.object.format() == object::BinaryFormat::MachO { + id.name().replace('.', "__") // machO expects __debug_info instead of .debug_info + } else { + id.name().to_string() + } + .into_bytes(); + + let segment = self.object.segment_name(StandardSegment::Debug).to_vec(); + // FIXME use SHT_X86_64_UNWIND for .eh_frame + let section_id = self.object.add_section( + segment, + name.clone(), + if id == SectionId::EhFrame { + SectionKind::ReadOnlyData + } else { + SectionKind::Debug + }, + ); + self.object + .section_mut(section_id) + .set_data(data, if id == SectionId::EhFrame { 8 } else { 1 }); + let symbol_id = self.object.section_symbol(section_id); + (section_id, symbol_id) + } + + fn add_debug_reloc( + &mut self, + section_map: &FxHashMap<SectionId, Self::SectionId>, + from: &Self::SectionId, + reloc: &DebugReloc, + ) { + let (symbol, symbol_offset) = match reloc.name { + DebugRelocName::Section(id) => (section_map.get(&id).unwrap().1, 0), + DebugRelocName::Symbol(id) => { + let symbol_id = self.function_symbol(FuncId::from_u32(id.try_into().unwrap())); + self.object + .symbol_section_and_offset(symbol_id) + .expect("Debug reloc for undef sym???") + } + }; + self.object + .add_relocation( + from.0, + Relocation { + offset: u64::from(reloc.offset), + symbol, + kind: reloc.kind, + encoding: RelocationEncoding::Generic, + size: reloc.size * 8, + addend: i64::try_from(symbol_offset).unwrap() + reloc.addend, + }, + ) + .unwrap(); + } +} + +// FIXME remove once atomic instructions are implemented in Cranelift. +pub(crate) trait AddConstructor { + fn add_constructor(&mut self, func_id: FuncId); +} + +impl AddConstructor for ObjectProduct { + fn add_constructor(&mut self, func_id: FuncId) { + let symbol = self.function_symbol(func_id); + let segment = self + .object + .segment_name(object::write::StandardSegment::Data); + let init_array_section = + self.object + .add_section(segment.to_vec(), b".init_array".to_vec(), SectionKind::Data); + let address_size = self + .object + .architecture() + .address_size() + .expect("address_size must be known") + .bytes(); + self.object.append_section_data( + init_array_section, + &std::iter::repeat(0) + .take(address_size.into()) + .collect::<Vec<u8>>(), + 8, + ); + self.object + .add_relocation( + init_array_section, + object::write::Relocation { + offset: 0, + size: address_size * 8, + kind: RelocationKind::Absolute, + encoding: RelocationEncoding::Generic, + symbol, + addend: 0, + }, + ) + .unwrap(); + } +} + +pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> { + let triple = crate::build_isa(sess, true).triple().clone(); + + let binary_format = match triple.binary_format { + target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, + target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff, + target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO, + binary_format => sess.fatal(&format!("binary format {} is unsupported", binary_format)), + }; + let architecture = match triple.architecture { + target_lexicon::Architecture::X86_32(_) => object::Architecture::I386, + target_lexicon::Architecture::X86_64 => object::Architecture::X86_64, + target_lexicon::Architecture::Arm(_) => object::Architecture::Arm, + target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64, + architecture => sess.fatal(&format!( + "target architecture {:?} is unsupported", + architecture, + )), + }; + let endian = match triple.endianness().unwrap() { + target_lexicon::Endianness::Little => object::Endianness::Little, + target_lexicon::Endianness::Big => object::Endianness::Big, + }; + + let mut metadata_object = object::write::Object::new(binary_format, architecture, endian); + metadata_object.add_file_symbol(name.as_bytes().to_vec()); + f(&mut metadata_object); + metadata_object.write().unwrap() +} + +pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { + let mut builder = ObjectBuilder::new( + crate::build_isa(sess, true), + name + ".o", + cranelift_module::default_libcall_names(), + ) + .unwrap(); + if std::env::var("CG_CLIF_FUNCTION_SECTIONS").is_ok() { + builder.per_function_section(true); + } + let module = ObjectModule::new(builder); + module +} diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs new file mode 100644 index 00000000000..fa9b8853d39 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -0,0 +1,1020 @@ +//! Codegen of a single function + +use rustc_index::vec::IndexVec; +use rustc_middle::ty::adjustment::PointerCast; + +use crate::prelude::*; + +pub(crate) fn trans_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx, impl Module>, + instance: Instance<'tcx>, + linkage: Linkage, +) { + let tcx = cx.tcx; + + let mir = tcx.instance_mir(instance.def); + + // Declare function + let (name, sig) = get_function_name_and_sig(tcx, cx.module.isa().triple(), instance, false); + let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); + + cx.cached_context.clear(); + + // Make the FunctionBuilder + let mut func_ctx = FunctionBuilderContext::new(); + let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new()); + func.name = ExternalName::user(0, func_id.as_u32()); + func.signature = sig; + func.collect_debug_info(); + + let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx); + + // Predefine blocks + let start_block = bcx.create_block(); + let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len()) + .map(|_| bcx.create_block()) + .collect(); + + // Make FunctionCx + let pointer_type = cx.module.target_config().pointer_type(); + let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); + + let mut fx = FunctionCx { + cx, + tcx, + pointer_type, + + instance, + mir, + + bcx, + block_map, + local_map: IndexVec::with_capacity(mir.local_decls.len()), + caller_location: None, // set by `codegen_fn_prelude` + cold_blocks: EntitySet::new(), + + clif_comments, + source_info_set: indexmap::IndexSet::new(), + next_ssa_var: 0, + + inline_asm_index: 0, + }; + + let arg_uninhabited = fx.mir.args_iter().any(|arg| { + fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)) + .abi + .is_uninhabited() + }); + + if arg_uninhabited { + fx.bcx + .append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument"); + } else { + tcx.sess.time("codegen clif ir", || { + tcx.sess.time("codegen prelude", || { + crate::abi::codegen_fn_prelude(&mut fx, start_block) + }); + codegen_fn_content(&mut fx); + }); + } + + // Recover all necessary data from fx, before accessing func will prevent future access to it. + let instance = fx.instance; + let mut clif_comments = fx.clif_comments; + let source_info_set = fx.source_info_set; + let local_map = fx.local_map; + let cold_blocks = fx.cold_blocks; + + // Store function in context + let context = &mut cx.cached_context; + context.func = func; + + crate::pretty_clif::write_clif_file(tcx, "unopt", None, instance, &context, &clif_comments); + + // Verify function + verify_func(tcx, &clif_comments, &context.func); + + // Perform rust specific optimizations + tcx.sess.time("optimize clif ir", || { + crate::optimize::optimize_function( + tcx, + instance, + context, + &cold_blocks, + &mut clif_comments, + ); + }); + + // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` + // instruction, which doesn't have an encoding. + context.compute_cfg(); + context.compute_domtree(); + context.eliminate_unreachable_code(cx.module.isa()).unwrap(); + context.dce(cx.module.isa()).unwrap(); + + // Define function + let module = &mut cx.module; + tcx.sess.time("define function", || { + module + .define_function( + func_id, + context, + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap() + }); + + // Write optimized function to file for debugging + crate::pretty_clif::write_clif_file( + tcx, + "opt", + Some(cx.module.isa()), + instance, + &context, + &clif_comments, + ); + + // Define debuginfo for function + let isa = cx.module.isa(); + let debug_context = &mut cx.debug_context; + let unwind_context = &mut cx.unwind_context; + tcx.sess.time("generate debug info", || { + if let Some(debug_context) = debug_context { + debug_context.define_function( + instance, + func_id, + &name, + isa, + context, + &source_info_set, + local_map, + ); + } + unwind_context.add_function(func_id, &context, isa); + }); + + // Clear context to make it usable for the next function + context.clear(); +} + +pub(crate) fn verify_func( + tcx: TyCtxt<'_>, + writer: &crate::pretty_clif::CommentWriter, + func: &Function, +) { + tcx.sess.time("verify clif ir", || { + let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); + match cranelift_codegen::verify_function(&func, &flags) { + Ok(_) => {} + Err(err) => { + tcx.sess.err(&format!("{:?}", err)); + let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( + &func, + None, + Some(Box::new(writer)), + err, + ); + tcx.sess + .fatal(&format!("cranelift verify error:\n{}", pretty_error)); + } + } + }); +} + +fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) { + crate::constant::check_constants(fx); + + for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { + let block = fx.get_block(bb); + fx.bcx.switch_to_block(block); + + if bb_data.is_cleanup { + // Unwinding after panicking is not supported + continue; + + // FIXME once unwinding is supported uncomment next lines + // // Unwinding is unlikely to happen, so mark cleanup block's as cold. + // fx.cold_blocks.insert(block); + } + + fx.bcx.ins().nop(); + for stmt in &bb_data.statements { + fx.set_debug_loc(stmt.source_info); + trans_stmt(fx, block, stmt); + } + + #[cfg(debug_assertions)] + { + let mut terminator_head = "\n".to_string(); + bb_data + .terminator() + .kind + .fmt_head(&mut terminator_head) + .unwrap(); + let inst = fx.bcx.func.layout.last_inst(block).unwrap(); + fx.add_comment(inst, terminator_head); + } + + fx.set_debug_loc(bb_data.terminator().source_info); + + match &bb_data.terminator().kind { + TerminatorKind::Goto { target } => { + if let TerminatorKind::Return = fx.mir[*target].terminator().kind { + let mut can_immediately_return = true; + for stmt in &fx.mir[*target].statements { + if let StatementKind::StorageDead(_) = stmt.kind { + } else { + // FIXME Can sometimes happen, see rust-lang/rust#70531 + can_immediately_return = false; + break; + } + } + + if can_immediately_return { + crate::abi::codegen_return(fx); + continue; + } + } + + let block = fx.get_block(*target); + fx.bcx.ins().jump(block, &[]); + } + TerminatorKind::Return => { + crate::abi::codegen_return(fx); + } + TerminatorKind::Assert { + cond, + expected, + msg, + target, + cleanup: _, + } => { + if !fx.tcx.sess.overflow_checks() { + if let mir::AssertKind::OverflowNeg(_) = *msg { + let target = fx.get_block(*target); + fx.bcx.ins().jump(target, &[]); + continue; + } + } + let cond = trans_operand(fx, cond).load_scalar(fx); + + let target = fx.get_block(*target); + let failure = fx.bcx.create_block(); + fx.cold_blocks.insert(failure); + + if *expected { + fx.bcx.ins().brz(cond, failure, &[]); + } else { + fx.bcx.ins().brnz(cond, failure, &[]); + }; + fx.bcx.ins().jump(target, &[]); + + fx.bcx.switch_to_block(failure); + fx.bcx.ins().nop(); + + match msg { + AssertKind::BoundsCheck { ref len, ref index } => { + let len = trans_operand(fx, len).load_scalar(fx); + let index = trans_operand(fx, index).load_scalar(fx); + let location = fx + .get_caller_location(bb_data.terminator().source_info.span) + .load_scalar(fx); + + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicBoundsCheck, + &[index, len, location], + bb_data.terminator().source_info.span, + ); + } + _ => { + let msg_str = msg.description(); + codegen_panic(fx, msg_str, bb_data.terminator().source_info.span); + } + } + } + + TerminatorKind::SwitchInt { + discr, + switch_ty, + targets, + } => { + let discr = trans_operand(fx, discr).load_scalar(fx); + + if switch_ty.kind() == fx.tcx.types.bool.kind() { + assert_eq!(targets.iter().count(), 1); + let (then_value, then_block) = targets.iter().next().unwrap(); + let then_block = fx.get_block(then_block); + let else_block = fx.get_block(targets.otherwise()); + let test_zero = match then_value { + 0 => true, + 1 => false, + _ => unreachable!("{:?}", targets), + }; + + let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); + let (discr, is_inverted) = + crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr); + let test_zero = if is_inverted { !test_zero } else { test_zero }; + let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); + let discr = + crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr); + if test_zero { + fx.bcx.ins().brz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } else { + fx.bcx.ins().brnz(discr, then_block, &[]); + fx.bcx.ins().jump(else_block, &[]); + } + } else { + let mut switch = ::cranelift_frontend::Switch::new(); + for (value, block) in targets.iter() { + let block = fx.get_block(block); + switch.set_entry(value, block); + } + let otherwise_block = fx.get_block(targets.otherwise()); + switch.emit(&mut fx.bcx, discr, otherwise_block); + } + } + TerminatorKind::Call { + func, + args, + destination, + fn_span, + cleanup: _, + from_hir_call: _, + } => { + fx.tcx.sess.time("codegen call", || { + crate::abi::codegen_terminator_call( + fx, + *fn_span, + block, + func, + args, + *destination, + ) + }); + } + TerminatorKind::InlineAsm { + template, + operands, + options, + destination, + line_spans: _, + } => { + crate::inline_asm::codegen_inline_asm( + fx, + bb_data.terminator().source_info.span, + template, + operands, + *options, + ); + + match *destination { + Some(destination) => { + let destination_block = fx.get_block(destination); + fx.bcx.ins().jump(destination_block, &[]); + } + None => { + crate::trap::trap_unreachable( + fx, + "[corruption] Returned from noreturn inline asm", + ); + } + } + } + TerminatorKind::Resume | TerminatorKind::Abort => { + trap_unreachable(fx, "[corruption] Unwinding bb reached."); + } + TerminatorKind::Unreachable => { + trap_unreachable(fx, "[corruption] Hit unreachable code."); + } + TerminatorKind::Yield { .. } + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } + | TerminatorKind::DropAndReplace { .. } + | TerminatorKind::GeneratorDrop => { + bug!("shouldn't exist at trans {:?}", bb_data.terminator()); + } + TerminatorKind::Drop { + place, + target, + unwind: _, + } => { + let drop_place = trans_place(fx, *place); + crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place); + + let target_block = fx.get_block(*target); + fx.bcx.ins().jump(target_block, &[]); + } + }; + } + + fx.bcx.seal_all_blocks(); + fx.bcx.finalize(); +} + +fn trans_stmt<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + #[allow(unused_variables)] cur_block: Block, + stmt: &Statement<'tcx>, +) { + let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); + + fx.set_debug_loc(stmt.source_info); + + #[cfg(false_debug_assertions)] + match &stmt.kind { + StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful + _ => { + let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); + fx.add_comment(inst, format!("{:?}", stmt)); + } + } + + match &stmt.kind { + StatementKind::SetDiscriminant { + place, + variant_index, + } => { + let place = trans_place(fx, **place); + crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); + } + StatementKind::Assign(to_place_and_rval) => { + let lval = trans_place(fx, to_place_and_rval.0); + let dest_layout = lval.layout(); + match &to_place_and_rval.1 { + Rvalue::Use(operand) => { + let val = trans_operand(fx, operand); + lval.write_cvalue(fx, val); + } + Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { + let place = trans_place(fx, *place); + let ref_ = place.place_ref(fx, lval.layout()); + lval.write_cvalue(fx, ref_); + } + Rvalue::ThreadLocalRef(def_id) => { + let val = crate::constant::codegen_tls_ref(fx, *def_id, lval.layout()); + lval.write_cvalue(fx, val); + } + Rvalue::BinaryOp(bin_op, lhs, rhs) => { + let lhs = trans_operand(fx, lhs); + let rhs = trans_operand(fx, rhs); + + let res = crate::num::codegen_binop(fx, *bin_op, lhs, rhs); + lval.write_cvalue(fx, res); + } + Rvalue::CheckedBinaryOp(bin_op, lhs, rhs) => { + let lhs = trans_operand(fx, lhs); + let rhs = trans_operand(fx, rhs); + + let res = if !fx.tcx.sess.overflow_checks() { + let val = + crate::num::trans_int_binop(fx, *bin_op, lhs, rhs).load_scalar(fx); + let is_overflow = fx.bcx.ins().iconst(types::I8, 0); + CValue::by_val_pair(val, is_overflow, lval.layout()) + } else { + crate::num::trans_checked_int_binop(fx, *bin_op, lhs, rhs) + }; + + lval.write_cvalue(fx, res); + } + Rvalue::UnaryOp(un_op, operand) => { + let operand = trans_operand(fx, operand); + let layout = operand.layout(); + let val = operand.load_scalar(fx); + let res = match un_op { + UnOp::Not => match layout.ty.kind() { + ty::Bool => { + let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); + CValue::by_val(fx.bcx.ins().bint(types::I8, res), layout) + } + ty::Uint(_) | ty::Int(_) => { + CValue::by_val(fx.bcx.ins().bnot(val), layout) + } + _ => unreachable!("un op Not for {:?}", layout.ty), + }, + UnOp::Neg => match layout.ty.kind() { + ty::Int(IntTy::I128) => { + // FIXME remove this case once ineg.i128 works + let zero = CValue::const_val(fx, layout, 0); + crate::num::trans_int_binop(fx, BinOp::Sub, zero, operand) + } + ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), + ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), + _ => unreachable!("un op Neg for {:?}", layout.ty), + }, + }; + lval.write_cvalue(fx, res); + } + Rvalue::Cast(CastKind::Pointer(PointerCast::ReifyFnPointer), operand, to_ty) => { + let from_ty = fx.monomorphize(&operand.ty(&fx.mir.local_decls, fx.tcx)); + let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + match *from_ty.kind() { + ty::FnDef(def_id, substs) => { + let func_ref = fx.get_function_ref( + Instance::resolve_for_fn_ptr( + fx.tcx, + ParamEnv::reveal_all(), + def_id, + substs, + ) + .unwrap() + .polymorphize(fx.tcx), + ); + let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); + lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout)); + } + _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), + } + } + Rvalue::Cast(CastKind::Pointer(PointerCast::UnsafeFnPointer), operand, to_ty) + | Rvalue::Cast(CastKind::Pointer(PointerCast::MutToConstPointer), operand, to_ty) + | Rvalue::Cast(CastKind::Pointer(PointerCast::ArrayToPointer), operand, to_ty) => { + let to_layout = fx.layout_of(fx.monomorphize(to_ty)); + let operand = trans_operand(fx, operand); + lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); + } + Rvalue::Cast(CastKind::Misc, operand, to_ty) => { + let operand = trans_operand(fx, operand); + let from_ty = operand.layout().ty; + let to_ty = fx.monomorphize(to_ty); + + fn is_fat_ptr<'tcx>( + fx: &FunctionCx<'_, 'tcx, impl Module>, + ty: Ty<'tcx>, + ) -> bool { + ty.builtin_deref(true) + .map( + |ty::TypeAndMut { + ty: pointee_ty, + mutbl: _, + }| { + has_ptr_meta(fx.tcx, pointee_ty) + }, + ) + .unwrap_or(false) + } + + if is_fat_ptr(fx, from_ty) { + if is_fat_ptr(fx, to_ty) { + // fat-ptr -> fat-ptr + lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout)); + } else { + // fat-ptr -> thin-ptr + let (ptr, _extra) = operand.load_scalar_pair(fx); + lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout)) + } + } else if let ty::Adt(adt_def, _substs) = from_ty.kind() { + // enum -> discriminant value + assert!(adt_def.is_enum()); + match to_ty.kind() { + ty::Uint(_) | ty::Int(_) => {} + _ => unreachable!("cast adt {} -> {}", from_ty, to_ty), + } + + use rustc_target::abi::{Int, TagEncoding, Variants}; + + match &operand.layout().variants { + Variants::Single { index } => { + let discr = operand + .layout() + .ty + .discriminant_for_variant(fx.tcx, *index) + .unwrap(); + let discr = if discr.ty.is_signed() { + rustc_middle::mir::interpret::sign_extend( + discr.val, + fx.layout_of(discr.ty).size, + ) + } else { + discr.val + }; + + let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr); + lval.write_cvalue(fx, discr); + } + Variants::Multiple { + tag, + tag_field, + tag_encoding: TagEncoding::Direct, + variants: _, + } => { + let cast_to = fx.clif_type(dest_layout.ty).unwrap(); + + // Read the tag/niche-encoded discriminant from memory. + let encoded_discr = + operand.value_field(fx, mir::Field::new(*tag_field)); + let encoded_discr = encoded_discr.load_scalar(fx); + + // Decode the discriminant (specifically if it's niche-encoded). + let signed = match tag.value { + Int(_, signed) => signed, + _ => false, + }; + let val = clif_intcast(fx, encoded_discr, cast_to, signed); + let val = CValue::by_val(val, dest_layout); + lval.write_cvalue(fx, val); + } + Variants::Multiple { .. } => unreachable!(), + } + } else { + let to_clif_ty = fx.clif_type(to_ty).unwrap(); + let from = operand.load_scalar(fx); + + let res = clif_int_or_float_cast( + fx, + from, + type_sign(from_ty), + to_clif_ty, + type_sign(to_ty), + ); + lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); + } + } + Rvalue::Cast( + CastKind::Pointer(PointerCast::ClosureFnPointer(_)), + operand, + _to_ty, + ) => { + let operand = trans_operand(fx, operand); + match *operand.layout().ty.kind() { + ty::Closure(def_id, substs) => { + let instance = Instance::resolve_closure( + fx.tcx, + def_id, + substs, + ty::ClosureKind::FnOnce, + ) + .polymorphize(fx.tcx); + let func_ref = fx.get_function_ref(instance); + let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); + lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout())); + } + _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty), + } + } + Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), operand, _to_ty) => { + let operand = trans_operand(fx, operand); + operand.unsize_value(fx, lval); + } + Rvalue::Discriminant(place) => { + let place = trans_place(fx, *place); + let value = place.to_cvalue(fx); + let discr = + crate::discriminant::codegen_get_discriminant(fx, value, dest_layout); + lval.write_cvalue(fx, discr); + } + Rvalue::Repeat(operand, times) => { + let operand = trans_operand(fx, operand); + let times = fx + .monomorphize(times) + .eval(fx.tcx, ParamEnv::reveal_all()) + .val + .try_to_bits(fx.tcx.data_layout.pointer_size) + .unwrap(); + if fx.clif_type(operand.layout().ty) == Some(types::I8) { + let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); + // FIXME use emit_small_memset where possible + let addr = lval.to_ptr().get_addr(fx); + let val = operand.load_scalar(fx); + fx.bcx + .call_memset(fx.cx.module.target_config(), addr, val, times); + } else { + let loop_block = fx.bcx.create_block(); + let loop_block2 = fx.bcx.create_block(); + let done_block = fx.bcx.create_block(); + let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); + let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); + fx.bcx.ins().jump(loop_block, &[zero]); + + fx.bcx.switch_to_block(loop_block); + let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); + fx.bcx.ins().brnz(done, done_block, &[]); + fx.bcx.ins().jump(loop_block2, &[]); + + fx.bcx.switch_to_block(loop_block2); + let to = lval.place_index(fx, index); + to.write_cvalue(fx, operand); + let index = fx.bcx.ins().iadd_imm(index, 1); + fx.bcx.ins().jump(loop_block, &[index]); + + fx.bcx.switch_to_block(done_block); + fx.bcx.ins().nop(); + } + } + Rvalue::Len(place) => { + let place = trans_place(fx, *place); + let usize_layout = fx.layout_of(fx.tcx.types.usize); + let len = codegen_array_len(fx, place); + lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); + } + Rvalue::NullaryOp(NullOp::Box, content_ty) => { + let usize_type = fx.clif_type(fx.tcx.types.usize).unwrap(); + let content_ty = fx.monomorphize(content_ty); + let layout = fx.layout_of(content_ty); + let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64); + let llalign = fx + .bcx + .ins() + .iconst(usize_type, layout.align.abi.bytes() as i64); + let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty)); + + // Allocate space: + let def_id = match fx + .tcx + .lang_items() + .require(rustc_hir::LangItem::ExchangeMalloc) + { + Ok(id) => id, + Err(s) => { + fx.tcx + .sess + .fatal(&format!("allocation of `{}` {}", box_layout.ty, s)); + } + }; + let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); + let func_ref = fx.get_function_ref(instance); + let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]); + let ptr = fx.bcx.inst_results(call)[0]; + lval.write_cvalue(fx, CValue::by_val(ptr, box_layout)); + } + Rvalue::NullaryOp(NullOp::SizeOf, ty) => { + assert!(lval + .layout() + .ty + .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())); + let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes(); + let val = + CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into()); + lval.write_cvalue(fx, val); + } + Rvalue::Aggregate(kind, operands) => match **kind { + AggregateKind::Array(_ty) => { + for (i, operand) in operands.into_iter().enumerate() { + let operand = trans_operand(fx, operand); + let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64); + let to = lval.place_index(fx, index); + to.write_cvalue(fx, operand); + } + } + _ => unreachable!("shouldn't exist at trans {:?}", to_place_and_rval.1), + }, + } + } + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Nop + | StatementKind::FakeRead(..) + | StatementKind::Retag { .. } + | StatementKind::AscribeUserType(..) => {} + + StatementKind::LlvmInlineAsm(asm) => { + use rustc_span::symbol::Symbol; + let LlvmInlineAsm { + asm, + outputs, + inputs, + } = &**asm; + let rustc_hir::LlvmInlineAsmInner { + asm: asm_code, // Name + outputs: output_names, // Vec<LlvmInlineAsmOutput> + inputs: input_names, // Vec<Name> + clobbers, // Vec<Name> + volatile, // bool + alignstack, // bool + dialect: _, + asm_str_style: _, + } = asm; + match asm_code.as_str().trim() { + "" => { + // Black box + } + "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { + assert_eq!( + input_names, + &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")] + ); + assert_eq!(output_names.len(), 4); + for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]) + .iter() + .enumerate() + { + assert_eq!(&output_names[i].constraint.as_str(), c); + assert!(!output_names[i].is_rw); + assert!(!output_names[i].is_indirect); + } + + assert_eq!(clobbers, &[]); + + assert!(!volatile); + assert!(!alignstack); + + assert_eq!(inputs.len(), 2); + let leaf = trans_operand(fx, &inputs[0].1).load_scalar(fx); // %eax + let subleaf = trans_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx + + let (eax, ebx, ecx, edx) = + crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); + + assert_eq!(outputs.len(), 4); + trans_place(fx, outputs[0]) + .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + trans_place(fx, outputs[1]) + .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + trans_place(fx, outputs[2]) + .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + trans_place(fx, outputs[3]) + .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + } + "xgetbv" => { + assert_eq!(input_names, &[Symbol::intern("{ecx}")]); + + assert_eq!(output_names.len(), 2); + for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { + assert_eq!(&output_names[i].constraint.as_str(), c); + assert!(!output_names[i].is_rw); + assert!(!output_names[i].is_indirect); + } + + assert_eq!(clobbers, &[]); + + assert!(!volatile); + assert!(!alignstack); + + crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); + } + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + _ if fx + .tcx + .symbol_name(fx.instance) + .name + .starts_with("___chkstk") => + { + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } + _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); + } + // Used in sys::windows::abort_internal + "int $$0x29" => { + crate::trap::trap_unimplemented(fx, "Windows abort"); + } + _ => fx + .tcx + .sess + .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), + } + } + StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), + } +} + +fn codegen_array_len<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + place: CPlace<'tcx>, +) -> Value { + match *place.layout().ty.kind() { + ty::Array(_elem_ty, len) => { + let len = fx + .monomorphize(&len) + .eval(fx.tcx, ParamEnv::reveal_all()) + .eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64; + fx.bcx.ins().iconst(fx.pointer_type, len) + } + ty::Slice(_elem_ty) => place + .to_ptr_maybe_unsized() + .1 + .expect("Length metadata for slice place"), + _ => bug!("Rvalue::Len({:?})", place), + } +} + +pub(crate) fn trans_place<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + place: Place<'tcx>, +) -> CPlace<'tcx> { + let mut cplace = fx.get_local_place(place.local); + + for elem in place.projection { + match elem { + PlaceElem::Deref => { + cplace = cplace.place_deref(fx); + } + PlaceElem::Field(field, _ty) => { + cplace = cplace.place_field(fx, field); + } + PlaceElem::Index(local) => { + let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); + cplace = cplace.place_index(fx, index); + } + PlaceElem::ConstantIndex { + offset, + min_length: _, + from_end, + } => { + let offset: u64 = offset; + let index = if !from_end { + fx.bcx.ins().iconst(fx.pointer_type, offset as i64) + } else { + let len = codegen_array_len(fx, cplace); + fx.bcx.ins().iadd_imm(len, -(offset as i64)) + }; + cplace = cplace.place_index(fx, index); + } + PlaceElem::Subslice { from, to, from_end } => { + // These indices are generated by slice patterns. + // slice[from:-to] in Python terms. + + let from: u64 = from; + let to: u64 = to; + + match cplace.layout().ty.kind() { + ty::Array(elem_ty, _len) => { + assert!(!from_end, "array subslices are never `from_end`"); + let elem_layout = fx.layout_of(elem_ty); + let ptr = cplace.to_ptr(); + cplace = CPlace::for_ptr( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), + fx.layout_of(fx.tcx.mk_array(elem_ty, u64::from(to) - u64::from(from))), + ); + } + ty::Slice(elem_ty) => { + assert!(from_end, "slice subslices should be `from_end`"); + let elem_layout = fx.layout_of(elem_ty); + let (ptr, len) = cplace.to_ptr_maybe_unsized(); + let len = len.unwrap(); + cplace = CPlace::for_ptr_with_extra( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), + fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), + cplace.layout(), + ); + } + _ => unreachable!(), + } + } + PlaceElem::Downcast(_adt_def, variant) => { + cplace = cplace.downcast_variant(fx, variant); + } + } + } + + cplace +} + +pub(crate) fn trans_operand<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + operand: &Operand<'tcx>, +) -> CValue<'tcx> { + match operand { + Operand::Move(place) | Operand::Copy(place) => { + let cplace = trans_place(fx, *place); + cplace.to_cvalue(fx) + } + Operand::Constant(const_) => crate::constant::trans_constant(fx, const_), + } +} + +pub(crate) fn codegen_panic<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + msg_str: &str, + span: Span, +) { + let location = fx.get_caller_location(span).load_scalar(fx); + + let msg_ptr = fx.anonymous_str("assert", msg_str); + let msg_len = fx + .bcx + .ins() + .iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); + let args = [msg_ptr, msg_len, location]; + + codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span); +} + +pub(crate) fn codegen_panic_inner<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + lang_item: rustc_hir::LangItem, + args: &[Value], + span: Span, +) { + let def_id = fx + .tcx + .lang_items() + .require(lang_item) + .unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s)); + + let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); + let symbol_name = fx.tcx.symbol_name(instance).name; + + fx.lib_call( + &*symbol_name, + vec![fx.pointer_type, fx.pointer_type, fx.pointer_type], + vec![], + args, + ); + + crate::trap::trap_unreachable(fx, "panic lang item returned"); +} diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs new file mode 100644 index 00000000000..590c9ef0ce1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs @@ -0,0 +1,88 @@ +#![feature(rustc_private)] + +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_interface; +extern crate rustc_session; +extern crate rustc_target; + +use rustc_data_structures::profiling::print_time_passes_entry; +use rustc_interface::interface; +use rustc_session::config::ErrorOutputType; +use rustc_session::early_error; +use rustc_target::spec::PanicStrategy; + +#[derive(Default)] +pub struct CraneliftPassesCallbacks { + time_passes: bool, +} + +impl rustc_driver::Callbacks for CraneliftPassesCallbacks { + fn config(&mut self, config: &mut interface::Config) { + // If a --prints=... option has been given, we don't print the "total" + // time because it will mess up the --prints output. See #64339. + self.time_passes = config.opts.prints.is_empty() + && (config.opts.debugging_opts.time_passes || config.opts.debugging_opts.time); + + // FIXME workaround for an ICE + config.opts.debugging_opts.trim_diagnostic_paths = false; + + config.opts.cg.panic = Some(PanicStrategy::Abort); + config.opts.debugging_opts.panic_abort_tests = true; + config.opts.maybe_sysroot = Some( + std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join("build_sysroot") + .join("sysroot"), + ); + } +} + +fn main() { + let start = std::time::Instant::now(); + rustc_driver::init_rustc_env_logger(); + let mut callbacks = CraneliftPassesCallbacks::default(); + rustc_driver::install_ice_hook(); + let exit_code = rustc_driver::catch_with_exit_code(|| { + let mut use_jit = false; + + let mut args = std::env::args_os() + .enumerate() + .map(|(i, arg)| { + arg.into_string().unwrap_or_else(|arg| { + early_error( + ErrorOutputType::default(), + &format!("Argument {} is not valid Unicode: {:?}", i, arg), + ) + }) + }) + .filter(|arg| { + if arg == "--jit" { + use_jit = true; + false + } else { + true + } + }) + .collect::<Vec<_>>(); + if use_jit { + args.push("-Cprefer-dynamic".to_string()); + } + let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); + run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { + config: rustc_codegen_cranelift::BackendConfig { use_jit }, + }) + }))); + run_compiler.run() + }); + // The extra `\t` is necessary to align this label with the others. + print_time_passes_entry(callbacks.time_passes, "\ttotal", start.elapsed()); + std::process::exit(exit_code) +} diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs new file mode 100644 index 00000000000..c207d98d6c1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs @@ -0,0 +1,106 @@ +//! The only difference between this and cg_clif.rs is that this binary defaults to using cg_llvm +//! instead of cg_clif and requires `--clif` to use cg_clif and that this binary doesn't have JIT +//! support. +//! This is necessary as with Cargo `RUSTC` applies to both target crates and host crates. The host +//! crates must be built with cg_llvm as we are currently building a sysroot for cg_clif. +//! `RUSTFLAGS` however is only applied to target crates, so `--clif` would only be passed to the +//! target crates. + +#![feature(rustc_private)] + +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_interface; +extern crate rustc_session; +extern crate rustc_target; + +use std::path::PathBuf; + +use rustc_interface::interface; +use rustc_session::config::ErrorOutputType; +use rustc_session::early_error; +use rustc_target::spec::PanicStrategy; + +fn find_sysroot() -> String { + // Taken from https://github.com/Manishearth/rust-clippy/pull/911. + let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME")); + let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN")); + match (home, toolchain) { + (Some(home), Some(toolchain)) => format!("{}/toolchains/{}", home, toolchain), + _ => option_env!("RUST_SYSROOT") + .expect("need to specify RUST_SYSROOT env var or use rustup or multirust") + .to_owned(), + } +} + +pub struct CraneliftPassesCallbacks { + use_clif: bool, +} + +impl rustc_driver::Callbacks for CraneliftPassesCallbacks { + fn config(&mut self, config: &mut interface::Config) { + if !self.use_clif { + config.opts.maybe_sysroot = Some(PathBuf::from(find_sysroot())); + return; + } + + // FIXME workaround for an ICE + config.opts.debugging_opts.trim_diagnostic_paths = false; + + config.opts.cg.panic = Some(PanicStrategy::Abort); + config.opts.debugging_opts.panic_abort_tests = true; + config.opts.maybe_sysroot = Some( + std::env::current_exe() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .parent() + .unwrap() + .join("build_sysroot") + .join("sysroot"), + ); + } +} + +fn main() { + rustc_driver::init_rustc_env_logger(); + rustc_driver::install_ice_hook(); + let exit_code = rustc_driver::catch_with_exit_code(|| { + let mut use_clif = false; + + let args = std::env::args_os() + .enumerate() + .map(|(i, arg)| { + arg.into_string().unwrap_or_else(|arg| { + early_error( + ErrorOutputType::default(), + &format!("Argument {} is not valid Unicode: {:?}", i, arg), + ) + }) + }) + .filter(|arg| { + if arg == "--clif" { + use_clif = true; + false + } else { + true + } + }) + .collect::<Vec<_>>(); + + let mut callbacks = CraneliftPassesCallbacks { use_clif }; + + let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks); + if use_clif { + run_compiler.set_make_codegen_backend(Some(Box::new(move |_| { + Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { + config: rustc_codegen_cranelift::BackendConfig { use_jit: false }, + }) + }))); + } + run_compiler.run() + }); + std::process::exit(exit_code) +} diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs new file mode 100644 index 00000000000..122a36b5bf7 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/cast.rs @@ -0,0 +1,201 @@ +//! Various number casting functions + +use crate::prelude::*; + +pub(crate) fn clif_intcast( + fx: &mut FunctionCx<'_, '_, impl Module>, + val: Value, + to: Type, + signed: bool, +) -> Value { + let from = fx.bcx.func.dfg.value_type(val); + match (from, to) { + // equal + (_, _) if from == to => val, + + // extend + (_, types::I128) => { + let lo = if from == types::I64 { + val + } else if signed { + fx.bcx.ins().sextend(types::I64, val) + } else { + fx.bcx.ins().uextend(types::I64, val) + }; + let hi = if signed { + fx.bcx.ins().sshr_imm(lo, 63) + } else { + fx.bcx.ins().iconst(types::I64, 0) + }; + fx.bcx.ins().iconcat(lo, hi) + } + (_, _) if to.wider_or_equal(from) => { + if signed { + fx.bcx.ins().sextend(to, val) + } else { + fx.bcx.ins().uextend(to, val) + } + } + + // reduce + (types::I128, _) => { + let (lsb, _msb) = fx.bcx.ins().isplit(val); + if to == types::I64 { + lsb + } else { + fx.bcx.ins().ireduce(to, lsb) + } + } + (_, _) => fx.bcx.ins().ireduce(to, val), + } +} + +pub(crate) fn clif_int_or_float_cast( + fx: &mut FunctionCx<'_, '_, impl Module>, + from: Value, + from_signed: bool, + to_ty: Type, + to_signed: bool, +) -> Value { + let from_ty = fx.bcx.func.dfg.value_type(from); + + if from_ty.is_int() && to_ty.is_int() { + // int-like -> int-like + clif_intcast( + fx, + from, + to_ty, + // This is correct as either from_signed == to_signed (=> this is trivially correct) + // Or from_clif_ty == to_clif_ty, which means this is a no-op. + from_signed, + ) + } else if from_ty.is_int() && to_ty.is_float() { + if from_ty == types::I128 { + // _______ss__f_ + // __float tisf: i128 -> f32 + // __float tidf: i128 -> f64 + // __floatuntisf: u128 -> f32 + // __floatuntidf: u128 -> f64 + + let name = format!( + "__float{sign}ti{flt}f", + sign = if from_signed { "" } else { "un" }, + flt = match to_ty { + types::F32 => "s", + types::F64 => "d", + _ => unreachable!("{:?}", to_ty), + }, + ); + + let from_rust_ty = if from_signed { + fx.tcx.types.i128 + } else { + fx.tcx.types.u128 + }; + + let to_rust_ty = match to_ty { + types::F32 => fx.tcx.types.f32, + types::F64 => fx.tcx.types.f64, + _ => unreachable!(), + }; + + return fx + .easy_call( + &name, + &[CValue::by_val(from, fx.layout_of(from_rust_ty))], + to_rust_ty, + ) + .load_scalar(fx); + } + + // int-like -> float + if from_signed { + fx.bcx.ins().fcvt_from_sint(to_ty, from) + } else { + fx.bcx.ins().fcvt_from_uint(to_ty, from) + } + } else if from_ty.is_float() && to_ty.is_int() { + if to_ty == types::I128 { + // _____sssf___ + // __fix sfti: f32 -> i128 + // __fix dfti: f64 -> i128 + // __fixunssfti: f32 -> u128 + // __fixunsdfti: f64 -> u128 + + let name = format!( + "__fix{sign}{flt}fti", + sign = if to_signed { "" } else { "uns" }, + flt = match from_ty { + types::F32 => "s", + types::F64 => "d", + _ => unreachable!("{:?}", to_ty), + }, + ); + + let from_rust_ty = match from_ty { + types::F32 => fx.tcx.types.f32, + types::F64 => fx.tcx.types.f64, + _ => unreachable!(), + }; + + let to_rust_ty = if to_signed { + fx.tcx.types.i128 + } else { + fx.tcx.types.u128 + }; + + return fx + .easy_call( + &name, + &[CValue::by_val(from, fx.layout_of(from_rust_ty))], + to_rust_ty, + ) + .load_scalar(fx); + } + + // float -> int-like + if to_ty == types::I8 || to_ty == types::I16 { + // FIXME implement fcvt_to_*int_sat.i8/i16 + let val = if to_signed { + fx.bcx.ins().fcvt_to_sint_sat(types::I32, from) + } else { + fx.bcx.ins().fcvt_to_uint_sat(types::I32, from) + }; + let (min, max) = match (to_ty, to_signed) { + (types::I8, false) => (0, i64::from(u8::MAX)), + (types::I16, false) => (0, i64::from(u16::MAX)), + (types::I8, true) => (i64::from(i8::MIN), i64::from(i8::MAX)), + (types::I16, true) => (i64::from(i16::MIN), i64::from(i16::MAX)), + _ => unreachable!(), + }; + let min_val = fx.bcx.ins().iconst(types::I32, min); + let max_val = fx.bcx.ins().iconst(types::I32, max); + + let val = if to_signed { + let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, min); + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, val, max); + let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, val); + fx.bcx.ins().select(has_overflow, max_val, bottom_capped) + } else { + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, val, max); + fx.bcx.ins().select(has_overflow, max_val, val) + }; + fx.bcx.ins().ireduce(to_ty, val) + } else { + if to_signed { + fx.bcx.ins().fcvt_to_sint_sat(to_ty, from) + } else { + fx.bcx.ins().fcvt_to_uint_sat(to_ty, from) + } + } + } else if from_ty.is_float() && to_ty.is_float() { + // float -> float + match (from_ty, to_ty) { + (types::F32, types::F64) => fx.bcx.ins().fpromote(types::F64, from), + (types::F64, types::F32) => fx.bcx.ins().fdemote(types::F32, from), + _ => from, + } + } else { + unreachable!("cast value from {:?} to {:?}", from_ty, to_ty); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs new file mode 100644 index 00000000000..e998403dea6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -0,0 +1,165 @@ +//! Replaces 128-bit operators with lang item calls where necessary + +use crate::prelude::*; + +pub(crate) fn maybe_codegen<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + checked: bool, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> Option<CValue<'tcx>> { + if lhs.layout().ty != fx.tcx.types.u128 && lhs.layout().ty != fx.tcx.types.i128 { + return None; + } + + let lhs_val = lhs.load_scalar(fx); + let rhs_val = rhs.load_scalar(fx); + + let is_signed = type_sign(lhs.layout().ty); + + match bin_op { + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { + assert!(!checked); + return None; + } + BinOp::Add | BinOp::Sub if !checked => return None, + BinOp::Add => { + let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); + return Some(if is_signed { + fx.easy_call("__rust_i128_addo", &[lhs, rhs], out_ty) + } else { + fx.easy_call("__rust_u128_addo", &[lhs, rhs], out_ty) + }); + } + BinOp::Sub => { + let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); + return Some(if is_signed { + fx.easy_call("__rust_i128_subo", &[lhs, rhs], out_ty) + } else { + fx.easy_call("__rust_u128_subo", &[lhs, rhs], out_ty) + }); + } + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), + BinOp::Mul => { + let res = if checked { + let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); + if is_signed { + fx.easy_call("__rust_i128_mulo", &[lhs, rhs], out_ty) + } else { + fx.easy_call("__rust_u128_mulo", &[lhs, rhs], out_ty) + } + } else { + let val_ty = if is_signed { + fx.tcx.types.i128 + } else { + fx.tcx.types.u128 + }; + fx.easy_call("__multi3", &[lhs, rhs], val_ty) + }; + return Some(res); + } + BinOp::Div => { + assert!(!checked); + if is_signed { + Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128)) + } else { + Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128)) + } + } + BinOp::Rem => { + assert!(!checked); + if is_signed { + Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128)) + } else { + Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128)) + } + } + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => { + assert!(!checked); + return None; + } + BinOp::Shl | BinOp::Shr => { + let is_overflow = if checked { + // rhs >= 128 + + // FIXME support non 128bit rhs + /*let (rhs_lsb, rhs_msb) = fx.bcx.ins().isplit(rhs_val); + let rhs_msb_gt_0 = fx.bcx.ins().icmp_imm(IntCC::NotEqual, rhs_msb, 0); + let rhs_lsb_ge_128 = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, rhs_lsb, 127); + let is_overflow = fx.bcx.ins().bor(rhs_msb_gt_0, rhs_lsb_ge_128);*/ + let is_overflow = fx.bcx.ins().bconst(types::B1, false); + + Some(fx.bcx.ins().bint(types::I8, is_overflow)) + } else { + None + }; + + // Optimize `val >> 64`, because compiler_builtins uses it to deconstruct an 128bit + // integer into its lsb and msb. + // https://github.com/rust-lang-nursery/compiler-builtins/blob/79a6a1603d5672cbb9187ff41ff4d9b5048ac1cb/src/int/mod.rs#L217 + if resolve_value_imm(fx.bcx.func, rhs_val) == Some(64) { + let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs_val); + let all_zeros = fx.bcx.ins().iconst(types::I64, 0); + let val = match (bin_op, is_signed) { + (BinOp::Shr, false) => { + let val = fx.bcx.ins().iconcat(lhs_msb, all_zeros); + Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.u128))) + } + (BinOp::Shr, true) => { + let sign = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, lhs_msb, 0); + let all_ones = fx.bcx.ins().iconst(types::I64, u64::MAX as i64); + let all_sign_bits = fx.bcx.ins().select(sign, all_zeros, all_ones); + + let val = fx.bcx.ins().iconcat(lhs_msb, all_sign_bits); + Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.i128))) + } + (BinOp::Shl, _) => { + let val_ty = if is_signed { + fx.tcx.types.i128 + } else { + fx.tcx.types.u128 + }; + let val = fx.bcx.ins().iconcat(all_zeros, lhs_lsb); + Some(CValue::by_val(val, fx.layout_of(val_ty))) + } + _ => None, + }; + if let Some(val) = val { + if let Some(is_overflow) = is_overflow { + let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); + let val = val.load_scalar(fx); + return Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty))); + } else { + return Some(val); + } + } + } + + let truncated_rhs = clif_intcast(fx, rhs_val, types::I32, false); + let truncated_rhs = CValue::by_val(truncated_rhs, fx.layout_of(fx.tcx.types.u32)); + let val = match (bin_op, is_signed) { + (BinOp::Shl, false) => { + fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.u128) + } + (BinOp::Shl, true) => { + fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.i128) + } + (BinOp::Shr, false) => { + fx.easy_call("__lshrti3", &[lhs, truncated_rhs], fx.tcx.types.u128) + } + (BinOp::Shr, true) => { + fx.easy_call("__ashrti3", &[lhs, truncated_rhs], fx.tcx.types.i128) + } + (_, _) => unreachable!(), + }; + if let Some(is_overflow) = is_overflow { + let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); + let val = val.load_scalar(fx); + Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty))) + } else { + Some(val) + } + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs new file mode 100644 index 00000000000..13c62add41a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -0,0 +1,446 @@ +use rustc_index::vec::IndexVec; +use rustc_target::abi::{Integer, Primitive}; +use rustc_target::spec::{HasTargetSpec, Target}; + +use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; + +use crate::prelude::*; + +pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { + match tcx.data_layout.pointer_size.bits() { + 16 => types::I16, + 32 => types::I32, + 64 => types::I64, + bits => bug!("ptr_sized_integer: unknown pointer bit size {}", bits), + } +} + +pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type { + match scalar.value { + Primitive::Int(int, _sign) => match int { + Integer::I8 => types::I8, + Integer::I16 => types::I16, + Integer::I32 => types::I32, + Integer::I64 => types::I64, + Integer::I128 => types::I128, + }, + Primitive::F32 => types::F32, + Primitive::F64 => types::F64, + Primitive::Pointer => pointer_ty(tcx), + } +} + +fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Type> { + Some(match ty.kind() { + ty::Bool => types::I8, + ty::Uint(size) => match size { + UintTy::U8 => types::I8, + UintTy::U16 => types::I16, + UintTy::U32 => types::I32, + UintTy::U64 => types::I64, + UintTy::U128 => types::I128, + UintTy::Usize => pointer_ty(tcx), + }, + ty::Int(size) => match size { + IntTy::I8 => types::I8, + IntTy::I16 => types::I16, + IntTy::I32 => types::I32, + IntTy::I64 => types::I64, + IntTy::I128 => types::I128, + IntTy::Isize => pointer_ty(tcx), + }, + ty::Char => types::I32, + ty::Float(size) => match size { + FloatTy::F32 => types::F32, + FloatTy::F64 => types::F64, + }, + ty::FnPtr(_) => pointer_ty(tcx), + ty::RawPtr(TypeAndMut { + ty: pointee_ty, + mutbl: _, + }) + | ty::Ref(_, pointee_ty, _) => { + if has_ptr_meta(tcx, pointee_ty) { + return None; + } else { + pointer_ty(tcx) + } + } + ty::Adt(adt_def, _) if adt_def.repr.simd() => { + let (element, count) = match &tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().abi + { + Abi::Vector { element, count } => (element.clone(), *count), + _ => unreachable!(), + }; + + match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) { + // Cranelift currently only implements icmp for 128bit vectors. + Some(vector_ty) if vector_ty.bits() == 128 => vector_ty, + _ => return None, + } + } + ty::Param(_) => bug!("ty param {:?}", ty), + _ => return None, + }) +} + +fn clif_pair_type_from_ty<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Option<(types::Type, types::Type)> { + Some(match ty.kind() { + ty::Tuple(substs) if substs.len() == 2 => { + let mut types = substs.types(); + let a = clif_type_from_ty(tcx, types.next().unwrap())?; + let b = clif_type_from_ty(tcx, types.next().unwrap())?; + if a.is_vector() || b.is_vector() { + return None; + } + (a, b) + } + ty::RawPtr(TypeAndMut { + ty: pointee_ty, + mutbl: _, + }) + | ty::Ref(_, pointee_ty, _) => { + if has_ptr_meta(tcx, pointee_ty) { + (pointer_ty(tcx), pointer_ty(tcx)) + } else { + return None; + } + } + _ => return None, + }) +} + +/// Is a pointer to this type a fat ptr? +pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + let ptr_ty = tcx.mk_ptr(TypeAndMut { + ty, + mutbl: rustc_hir::Mutability::Not, + }); + match &tcx + .layout_of(ParamEnv::reveal_all().and(ptr_ty)) + .unwrap() + .abi + { + Abi::Scalar(_) => false, + Abi::ScalarPair(_, _) => true, + abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi), + } +} + +pub(crate) fn codegen_icmp_imm( + fx: &mut FunctionCx<'_, '_, impl Module>, + intcc: IntCC, + lhs: Value, + rhs: i128, +) -> Value { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + if lhs_ty == types::I128 { + // FIXME legalize `icmp_imm.i128` in Cranelift + + let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs); + let (rhs_lsb, rhs_msb) = (rhs as u128 as u64 as i64, (rhs as u128 >> 64) as u64 as i64); + + match intcc { + IntCC::Equal => { + let lsb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_lsb, rhs_lsb); + let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); + fx.bcx.ins().band(lsb_eq, msb_eq) + } + IntCC::NotEqual => { + let lsb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_lsb, rhs_lsb); + let msb_ne = fx.bcx.ins().icmp_imm(IntCC::NotEqual, lhs_msb, rhs_msb); + fx.bcx.ins().bor(lsb_ne, msb_ne) + } + _ => { + // if msb_eq { + // lsb_cc + // } else { + // msb_cc + // } + + let msb_eq = fx.bcx.ins().icmp_imm(IntCC::Equal, lhs_msb, rhs_msb); + let lsb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_lsb, rhs_lsb); + let msb_cc = fx.bcx.ins().icmp_imm(intcc, lhs_msb, rhs_msb); + + fx.bcx.ins().select(msb_eq, lsb_cc, msb_cc) + } + } + } else { + let rhs = i64::try_from(rhs).expect("codegen_icmp_imm rhs out of range for <128bit int"); + fx.bcx.ins().icmp_imm(intcc, lhs, rhs) + } +} + +fn resolve_normal_value_imm(func: &Function, val: Value) -> Option<i64> { + if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) { + if let InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm, + } = func.dfg[inst] + { + Some(imm.into()) + } else { + None + } + } else { + None + } +} + +fn resolve_128bit_value_imm(func: &Function, val: Value) -> Option<u128> { + let (lsb, msb) = if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) { + if let InstructionData::Binary { + opcode: Opcode::Iconcat, + args: [lsb, msb], + } = func.dfg[inst] + { + (lsb, msb) + } else { + return None; + } + } else { + return None; + }; + + let lsb = u128::from(resolve_normal_value_imm(func, lsb)? as u64); + let msb = u128::from(resolve_normal_value_imm(func, msb)? as u64); + + Some(msb << 64 | lsb) +} + +pub(crate) fn resolve_value_imm(func: &Function, val: Value) -> Option<u128> { + if func.dfg.value_type(val) == types::I128 { + resolve_128bit_value_imm(func, val) + } else { + resolve_normal_value_imm(func, val).map(|imm| u128::from(imm as u64)) + } +} + +pub(crate) fn type_min_max_value( + bcx: &mut FunctionBuilder<'_>, + ty: Type, + signed: bool, +) -> (Value, Value) { + assert!(ty.is_int()); + + if ty == types::I128 { + if signed { + let min = i128::MIN as u128; + let min_lsb = bcx.ins().iconst(types::I64, min as u64 as i64); + let min_msb = bcx.ins().iconst(types::I64, (min >> 64) as u64 as i64); + let min = bcx.ins().iconcat(min_lsb, min_msb); + + let max = i128::MIN as u128; + let max_lsb = bcx.ins().iconst(types::I64, max as u64 as i64); + let max_msb = bcx.ins().iconst(types::I64, (max >> 64) as u64 as i64); + let max = bcx.ins().iconcat(max_lsb, max_msb); + + return (min, max); + } else { + let min_half = bcx.ins().iconst(types::I64, 0); + let min = bcx.ins().iconcat(min_half, min_half); + + let max_half = bcx.ins().iconst(types::I64, u64::MAX as i64); + let max = bcx.ins().iconcat(max_half, max_half); + + return (min, max); + } + } + + let min = match (ty, signed) { + (types::I8, false) | (types::I16, false) | (types::I32, false) | (types::I64, false) => { + 0i64 + } + (types::I8, true) => i64::from(i8::MIN), + (types::I16, true) => i64::from(i16::MIN), + (types::I32, true) => i64::from(i32::MIN), + (types::I64, true) => i64::MIN, + _ => unreachable!(), + }; + + let max = match (ty, signed) { + (types::I8, false) => i64::from(u8::MAX), + (types::I16, false) => i64::from(u16::MAX), + (types::I32, false) => i64::from(u32::MAX), + (types::I64, false) => u64::MAX as i64, + (types::I8, true) => i64::from(i8::MAX), + (types::I16, true) => i64::from(i16::MAX), + (types::I32, true) => i64::from(i32::MAX), + (types::I64, true) => i64::MAX, + _ => unreachable!(), + }; + + let (min, max) = (bcx.ins().iconst(ty, min), bcx.ins().iconst(ty, max)); + + (min, max) +} + +pub(crate) fn type_sign(ty: Ty<'_>) -> bool { + match ty.kind() { + ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) | ty::Char | ty::Uint(..) | ty::Bool => false, + ty::Int(..) => true, + ty::Float(..) => false, // `signed` is unused for floats + _ => panic!("{}", ty), + } +} + +pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx, M>, + pub(crate) tcx: TyCtxt<'tcx>, + pub(crate) pointer_type: Type, // Cached from module + + pub(crate) instance: Instance<'tcx>, + pub(crate) mir: &'tcx Body<'tcx>, + + pub(crate) bcx: FunctionBuilder<'clif>, + pub(crate) block_map: IndexVec<BasicBlock, Block>, + pub(crate) local_map: IndexVec<Local, CPlace<'tcx>>, + + /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. + pub(crate) caller_location: Option<CValue<'tcx>>, + + /// See [`crate::optimize::code_layout`] for more information. + pub(crate) cold_blocks: EntitySet<Block>, + + pub(crate) clif_comments: crate::pretty_clif::CommentWriter, + pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>, + + /// This should only be accessed by `CPlace::new_var`. + pub(crate) next_ssa_var: u32, + + pub(crate) inline_asm_index: u32, +} + +impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> { + type Ty = Ty<'tcx>; + type TyAndLayout = TyAndLayout<'tcx>; + + fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> { + assert!(!ty.still_further_specializable()); + self.tcx + .layout_of(ParamEnv::reveal_all().and(&ty)) + .unwrap_or_else(|e| { + if let layout::LayoutError::SizeOverflow(_) = e { + self.tcx.sess.fatal(&e.to_string()) + } else { + bug!("failed to get layout for `{}`: {}", ty, e) + } + }) + } +} + +impl<'tcx, M: Module> layout::HasTyCtxt<'tcx> for FunctionCx<'_, 'tcx, M> { + fn tcx<'b>(&'b self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx, M: Module> rustc_target::abi::HasDataLayout for FunctionCx<'_, 'tcx, M> { + fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout { + &self.tcx.data_layout + } +} + +impl<'tcx, M: Module> layout::HasParamEnv<'tcx> for FunctionCx<'_, 'tcx, M> { + fn param_env(&self) -> ParamEnv<'tcx> { + ParamEnv::reveal_all() + } +} + +impl<'tcx, M: Module> HasTargetSpec for FunctionCx<'_, 'tcx, M> { + fn target_spec(&self) -> &Target { + &self.tcx.sess.target + } +} + +impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> { + pub(crate) fn monomorphize<T>(&self, value: &T) -> T + where + T: TypeFoldable<'tcx> + Copy, + { + if let Some(substs) = self.instance.substs_for_mir_body() { + self.tcx + .subst_and_normalize_erasing_regions(substs, ty::ParamEnv::reveal_all(), value) + } else { + self.tcx + .normalize_erasing_regions(ty::ParamEnv::reveal_all(), *value) + } + } + + pub(crate) fn clif_type(&self, ty: Ty<'tcx>) -> Option<Type> { + clif_type_from_ty(self.tcx, ty) + } + + pub(crate) fn clif_pair_type(&self, ty: Ty<'tcx>) -> Option<(Type, Type)> { + clif_pair_type_from_ty(self.tcx, ty) + } + + pub(crate) fn get_block(&self, bb: BasicBlock) -> Block { + *self.block_map.get(bb).unwrap() + } + + pub(crate) fn get_local_place(&mut self, local: Local) -> CPlace<'tcx> { + *self.local_map.get(local).unwrap_or_else(|| { + panic!("Local {:?} doesn't exist", local); + }) + } + + pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) { + let (index, _) = self.source_info_set.insert_full(source_info); + self.bcx.set_srcloc(SourceLoc::new(index as u32)); + } + + pub(crate) fn get_caller_location(&mut self, span: Span) -> CValue<'tcx> { + if let Some(loc) = self.caller_location { + // `#[track_caller]` is used; return caller location instead of current location. + return loc; + } + + let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); + let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo()); + let const_loc = self.tcx.const_caller_location(( + rustc_span::symbol::Symbol::intern(&caller.file.name.to_string()), + caller.line as u32, + caller.col_display as u32 + 1, + )); + crate::constant::trans_const_value(self, const_loc, self.tcx.caller_location_ty()) + } + + pub(crate) fn triple(&self) -> &target_lexicon::Triple { + self.cx.module.isa().triple() + } + + pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + msg.hash(&mut hasher); + let msg_hash = hasher.finish(); + let mut data_ctx = DataContext::new(); + data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); + let msg_id = self + .cx + .module + .declare_data( + &format!("__{}_{:08x}", prefix, msg_hash), + Linkage::Local, + false, + false, + ) + .unwrap(); + + // Ignore DuplicateDefinition error, as the data will be the same + let _ = self.cx.module.define_data(msg_id, &data_ctx); + + let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + #[cfg(debug_assertions)] + { + self.add_comment(local_msg_id, msg); + } + self.bcx.ins().global_value(self.pointer_type, local_msg_id) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs new file mode 100644 index 00000000000..1b514958a48 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -0,0 +1,476 @@ +//! Handling of `static`s, `const`s and promoted allocations + +use rustc_span::DUMMY_SP; + +use rustc_data_structures::fx::FxHashSet; +use rustc_errors::ErrorReported; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::interpret::{ + read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Pointer, Scalar, +}; +use rustc_middle::ty::{Const, ConstKind}; + +use cranelift_codegen::ir::GlobalValueData; +use cranelift_module::*; + +use crate::prelude::*; + +#[derive(Default)] +pub(crate) struct ConstantCx { + todo: Vec<TodoItem>, + done: FxHashSet<DataId>, +} + +#[derive(Copy, Clone, Debug)] +enum TodoItem { + Alloc(AllocId), + Static(DefId), +} + +impl ConstantCx { + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut impl Module) { + //println!("todo {:?}", self.todo); + define_all_allocs(tcx, module, &mut self); + //println!("done {:?}", self.done); + self.done.clear(); + } +} + +pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) { + for constant in &fx.mir.required_consts { + let const_ = fx.monomorphize(&constant.literal); + match const_.val { + ConstKind::Value(_) => {} + ConstKind::Unevaluated(def, ref substs, promoted) => { + if let Err(err) = + fx.tcx + .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) + { + match err { + ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => { + fx.tcx + .sess + .span_err(constant.span, "erroneous constant encountered"); + } + ErrorHandled::TooGeneric => { + span_bug!( + constant.span, + "codgen encountered polymorphic constant: {:?}", + err + ); + } + } + } + } + ConstKind::Param(_) + | ConstKind::Infer(_) + | ConstKind::Bound(_, _) + | ConstKind::Placeholder(_) + | ConstKind::Error(_) => unreachable!("{:?}", const_), + } + } +} + +pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { + constants_cx.todo.push(TodoItem::Static(def_id)); +} + +pub(crate) fn codegen_tls_ref<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + def_id: DefId, + layout: TyAndLayout<'tcx>, +) -> CValue<'tcx> { + let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false); + let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id); + CValue::by_val(tls_ptr, layout) +} + +fn codegen_static_ref<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + def_id: DefId, + layout: TyAndLayout<'tcx>, +) -> CPlace<'tcx> { + let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false); + let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(local_data_id, format!("{:?}", def_id)); + let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); + assert!(!layout.is_unsized(), "unsized statics aren't supported"); + assert!( + matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}), + "tls static referenced without Rvalue::ThreadLocalRef" + ); + CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) +} + +pub(crate) fn trans_constant<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + constant: &Constant<'tcx>, +) -> CValue<'tcx> { + let const_ = fx.monomorphize(&constant.literal); + let const_val = match const_.val { + ConstKind::Value(const_val) => const_val, + ConstKind::Unevaluated(def, ref substs, promoted) if fx.tcx.is_static(def.did) => { + assert!(substs.is_empty()); + assert!(promoted.is_none()); + + return codegen_static_ref( + fx, + def.did, + fx.layout_of(fx.monomorphize(&constant.literal.ty)), + ) + .to_cvalue(fx); + } + ConstKind::Unevaluated(def, ref substs, promoted) => { + match fx + .tcx + .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) + { + Ok(const_val) => const_val, + Err(_) => { + if promoted.is_none() { + fx.tcx + .sess + .span_err(constant.span, "erroneous constant encountered"); + } + return crate::trap::trap_unreachable_ret_value( + fx, + fx.layout_of(const_.ty), + "erroneous constant encountered", + ); + } + } + } + ConstKind::Param(_) + | ConstKind::Infer(_) + | ConstKind::Bound(_, _) + | ConstKind::Placeholder(_) + | ConstKind::Error(_) => unreachable!("{:?}", const_), + }; + + trans_const_value(fx, const_val, const_.ty) +} + +pub(crate) fn trans_const_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + const_val: ConstValue<'tcx>, + ty: Ty<'tcx>, +) -> CValue<'tcx> { + let layout = fx.layout_of(ty); + assert!(!layout.is_unsized(), "sized const value"); + + if layout.is_zst() { + return CValue::by_ref( + crate::Pointer::const_addr(fx, i64::try_from(layout.align.pref.bytes()).unwrap()), + layout, + ); + } + + match const_val { + ConstValue::Scalar(x) => { + if fx.clif_type(layout.ty).is_none() { + let (size, align) = (layout.size, layout.align.pref); + let mut alloc = Allocation::from_bytes( + std::iter::repeat(0) + .take(size.bytes_usize()) + .collect::<Vec<u8>>(), + align, + ); + let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used + alloc.write_scalar(fx, ptr, x.into(), size).unwrap(); + let alloc = fx.tcx.intern_const_alloc(alloc); + return CValue::by_ref(pointer_for_allocation(fx, alloc), layout); + } + + match x { + Scalar::Raw { data, size } => { + assert_eq!(u64::from(size), layout.size.bytes()); + return CValue::const_val(fx, layout, data); + } + Scalar::Ptr(ptr) => { + let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); + let base_addr = match alloc_kind { + Some(GlobalAlloc::Memory(alloc)) => { + fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); + let data_id = data_id_for_alloc_id( + &mut fx.cx.module, + ptr.alloc_id, + alloc.mutability, + ); + let local_data_id = + fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + Some(GlobalAlloc::Function(instance)) => { + let func_id = + crate::abi::import_function(fx.tcx, &mut fx.cx.module, instance); + let local_func_id = + fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) + } + Some(GlobalAlloc::Static(def_id)) => { + assert!(fx.tcx.is_static(def_id)); + let data_id = + data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false); + let local_data_id = + fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(local_data_id, format!("{:?}", def_id)); + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) + } + None => bug!("missing allocation {:?}", ptr.alloc_id), + }; + let val = if ptr.offset.bytes() != 0 { + fx.bcx + .ins() + .iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap()) + } else { + base_addr + }; + return CValue::by_val(val, layout); + } + } + } + ConstValue::ByRef { alloc, offset } => CValue::by_ref( + pointer_for_allocation(fx, alloc) + .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()), + layout, + ), + ConstValue::Slice { data, start, end } => { + let ptr = pointer_for_allocation(fx, data) + .offset_i64(fx, i64::try_from(start).unwrap()) + .get_addr(fx); + let len = fx.bcx.ins().iconst( + fx.pointer_type, + i64::try_from(end.checked_sub(start).unwrap()).unwrap(), + ); + CValue::by_val_pair(ptr, len, layout) + } + } +} + +fn pointer_for_allocation<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + alloc: &'tcx Allocation, +) -> crate::pointer::Pointer { + let alloc_id = fx.tcx.create_memory_alloc(alloc); + fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); + let data_id = data_id_for_alloc_id(&mut fx.cx.module, alloc_id, alloc.mutability); + + let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); + crate::pointer::Pointer::new(global_ptr) +} + +fn data_id_for_alloc_id( + module: &mut impl Module, + alloc_id: AllocId, + mutability: rustc_hir::Mutability, +) -> DataId { + module + .declare_data( + &format!("__alloc_{:x}", alloc_id.0), + Linkage::Local, + mutability == rustc_hir::Mutability::Mut, + false, + ) + .unwrap() +} + +fn data_id_for_static( + tcx: TyCtxt<'_>, + module: &mut impl Module, + def_id: DefId, + definition: bool, +) -> DataId { + let rlinkage = tcx.codegen_fn_attrs(def_id).linkage; + let linkage = if definition { + crate::linkage::get_static_linkage(tcx, def_id) + } else { + if rlinkage == Some(rustc_middle::mir::mono::Linkage::ExternalWeak) + || rlinkage == Some(rustc_middle::mir::mono::Linkage::WeakAny) + { + Linkage::Preemptible + } else { + Linkage::Import + } + }; + + let instance = Instance::mono(tcx, def_id).polymorphize(tcx); + let symbol_name = tcx.symbol_name(instance).name; + let ty = instance.ty(tcx, ParamEnv::reveal_all()); + let is_mutable = if tcx.is_mutable_static(def_id) { + true + } else { + !ty.is_freeze(tcx.at(DUMMY_SP), ParamEnv::reveal_all()) + }; + let align = tcx + .layout_of(ParamEnv::reveal_all().and(ty)) + .unwrap() + .align + .pref + .bytes(); + + let attrs = tcx.codegen_fn_attrs(def_id); + + let data_id = module + .declare_data( + &*symbol_name, + linkage, + is_mutable, + attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), + ) + .unwrap(); + + if rlinkage.is_some() { + // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141 + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + + let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name); + let ref_data_id = module + .declare_data(&ref_name, Linkage::Local, false, false) + .unwrap(); + let mut data_ctx = DataContext::new(); + data_ctx.set_align(align); + let data = module.declare_data_in_data(data_id, &mut data_ctx); + data_ctx.define( + std::iter::repeat(0) + .take(pointer_ty(tcx).bytes() as usize) + .collect(), + ); + data_ctx.write_data_addr(0, data, 0); + match module.define_data(ref_data_id, &data_ctx) { + // Every time the static is referenced there will be another definition of this global, + // so duplicate definitions are expected and allowed. + Err(ModuleError::DuplicateDefinition(_)) => {} + res => res.unwrap(), + } + ref_data_id + } else { + data_id + } +} + +fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut ConstantCx) { + while let Some(todo_item) = cx.todo.pop() { + let (data_id, alloc, section_name) = match todo_item { + TodoItem::Alloc(alloc_id) => { + //println!("alloc_id {}", alloc_id); + let alloc = match tcx.get_global_alloc(alloc_id).unwrap() { + GlobalAlloc::Memory(alloc) => alloc, + GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), + }; + let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability); + (data_id, alloc, None) + } + TodoItem::Static(def_id) => { + //println!("static {:?}", def_id); + + let section_name = tcx + .codegen_fn_attrs(def_id) + .link_section + .map(|s| s.as_str()); + + let alloc = tcx.eval_static_initializer(def_id).unwrap(); + + let data_id = data_id_for_static(tcx, module, def_id, true); + (data_id, alloc, section_name) + } + }; + + //("data_id {}", data_id); + if cx.done.contains(&data_id) { + continue; + } + + let mut data_ctx = DataContext::new(); + data_ctx.set_align(alloc.align.bytes()); + + if let Some(section_name) = section_name { + // FIXME set correct segment for Mach-O files + data_ctx.set_segment_section("", &*section_name); + } + + let bytes = alloc + .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) + .to_vec(); + data_ctx.define(bytes.into_boxed_slice()); + + for &(offset, (_tag, reloc)) in alloc.relocations().iter() { + let addend = { + let endianness = tcx.data_layout.endian; + let offset = offset.bytes() as usize; + let ptr_size = tcx.data_layout.pointer_size; + let bytes = &alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..offset + ptr_size.bytes() as usize, + ); + read_target_uint(endianness, bytes).unwrap() + }; + + let reloc_target_alloc = tcx.get_global_alloc(reloc).unwrap(); + let data_id = match reloc_target_alloc { + GlobalAlloc::Function(instance) => { + assert_eq!(addend, 0); + let func_id = crate::abi::import_function(tcx, module, instance); + let local_func_id = module.declare_func_in_data(func_id, &mut data_ctx); + data_ctx.write_function_addr(offset.bytes() as u32, local_func_id); + continue; + } + GlobalAlloc::Memory(target_alloc) => { + cx.todo.push(TodoItem::Alloc(reloc)); + data_id_for_alloc_id(module, reloc, target_alloc.mutability) + } + GlobalAlloc::Static(def_id) => { + if tcx + .codegen_fn_attrs(def_id) + .flags + .contains(CodegenFnAttrFlags::THREAD_LOCAL) + { + tcx.sess.fatal(&format!( + "Allocation {:?} contains reference to TLS value {:?}", + alloc, def_id + )); + } + + // Don't push a `TodoItem::Static` here, as it will cause statics used by + // multiple crates to be duplicated between them. It isn't necessary anyway, + // as it will get pushed by `codegen_static` when necessary. + data_id_for_static(tcx, module, def_id, false) + } + }; + + let global_value = module.declare_data_in_data(data_id, &mut data_ctx); + data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); + } + + module.define_data(data_id, &data_ctx).unwrap(); + cx.done.insert(data_id); + } + + assert!(cx.todo.is_empty(), "{:?}", cx.todo); +} + +pub(crate) fn mir_operand_get_const_val<'tcx>( + fx: &FunctionCx<'_, 'tcx, impl Module>, + operand: &Operand<'tcx>, +) -> Option<&'tcx Const<'tcx>> { + match operand { + Operand::Copy(_) | Operand::Move(_) => None, + Operand::Constant(const_) => Some( + fx.monomorphize(&const_.literal) + .eval(fx.tcx, ParamEnv::reveal_all()), + ), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs new file mode 100644 index 00000000000..cf8fee2b1d1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs @@ -0,0 +1,204 @@ +//! Write the debuginfo into an object file. + +use rustc_data_structures::fx::FxHashMap; + +use gimli::write::{Address, AttributeValue, EndianVec, Result, Sections, Writer}; +use gimli::{RunTimeEndian, SectionId}; + +use crate::backend::WriteDebugInfo; + +use super::DebugContext; + +impl DebugContext<'_> { + pub(crate) fn emit<P: WriteDebugInfo>(&mut self, product: &mut P) { + let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone()); + let root = self.dwarf.unit.root(); + let root = self.dwarf.unit.get_mut(root); + root.set( + gimli::DW_AT_ranges, + AttributeValue::RangeListRef(unit_range_list_id), + ); + + let mut sections = Sections::new(WriterRelocate::new(self.endian)); + self.dwarf.write(&mut sections).unwrap(); + + let mut section_map = FxHashMap::default(); + let _: Result<()> = sections.for_each_mut(|id, section| { + if !section.writer.slice().is_empty() { + let section_id = product.add_debug_section(id, section.writer.take()); + section_map.insert(id, section_id); + } + Ok(()) + }); + + let _: Result<()> = sections.for_each(|id, section| { + if let Some(section_id) = section_map.get(&id) { + for reloc in §ion.relocs { + product.add_debug_reloc(§ion_map, section_id, reloc); + } + } + Ok(()) + }); + } +} + +#[derive(Clone)] +pub(crate) struct DebugReloc { + pub(crate) offset: u32, + pub(crate) size: u8, + pub(crate) name: DebugRelocName, + pub(crate) addend: i64, + pub(crate) kind: object::RelocationKind, +} + +#[derive(Clone)] +pub(crate) enum DebugRelocName { + Section(SectionId), + Symbol(usize), +} + +/// A [`Writer`] that collects all necessary relocations. +#[derive(Clone)] +pub(super) struct WriterRelocate { + pub(super) relocs: Vec<DebugReloc>, + pub(super) writer: EndianVec<RunTimeEndian>, +} + +impl WriterRelocate { + pub(super) fn new(endian: RunTimeEndian) -> Self { + WriterRelocate { + relocs: Vec::new(), + writer: EndianVec::new(endian), + } + } + + /// Perform the collected relocations to be usable for JIT usage. + #[cfg(feature = "jit")] + pub(super) fn relocate_for_jit( + mut self, + jit_product: &cranelift_simplejit::SimpleJITProduct, + ) -> Vec<u8> { + use std::convert::TryInto; + + for reloc in self.relocs.drain(..) { + match reloc.name { + super::DebugRelocName::Section(_) => unreachable!(), + super::DebugRelocName::Symbol(sym) => { + let addr = jit_product + .lookup_func(cranelift_module::FuncId::from_u32(sym.try_into().unwrap())); + let val = (addr as u64 as i64 + reloc.addend) as u64; + self.writer + .write_udata_at(reloc.offset as usize, val, reloc.size) + .unwrap(); + } + } + } + self.writer.into_vec() + } +} + +impl Writer for WriterRelocate { + type Endian = RunTimeEndian; + + fn endian(&self) -> Self::Endian { + self.writer.endian() + } + + fn len(&self) -> usize { + self.writer.len() + } + + fn write(&mut self, bytes: &[u8]) -> Result<()> { + self.writer.write(bytes) + } + + fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()> { + self.writer.write_at(offset, bytes) + } + + fn write_address(&mut self, address: Address, size: u8) -> Result<()> { + match address { + Address::Constant(val) => self.write_udata(val, size), + Address::Symbol { symbol, addend } => { + let offset = self.len() as u64; + self.relocs.push(DebugReloc { + offset: offset as u32, + size, + name: DebugRelocName::Symbol(symbol), + addend: addend as i64, + kind: object::RelocationKind::Absolute, + }); + self.write_udata(0, size) + } + } + } + + fn write_offset(&mut self, val: usize, section: SectionId, size: u8) -> Result<()> { + let offset = self.len() as u32; + self.relocs.push(DebugReloc { + offset, + size, + name: DebugRelocName::Section(section), + addend: val as i64, + kind: object::RelocationKind::Absolute, + }); + self.write_udata(0, size) + } + + fn write_offset_at( + &mut self, + offset: usize, + val: usize, + section: SectionId, + size: u8, + ) -> Result<()> { + self.relocs.push(DebugReloc { + offset: offset as u32, + size, + name: DebugRelocName::Section(section), + addend: val as i64, + kind: object::RelocationKind::Absolute, + }); + self.write_udata_at(offset, 0, size) + } + + fn write_eh_pointer(&mut self, address: Address, eh_pe: gimli::DwEhPe, size: u8) -> Result<()> { + match address { + // Address::Constant arm copied from gimli + Address::Constant(val) => { + // Indirect doesn't matter here. + let val = match eh_pe.application() { + gimli::DW_EH_PE_absptr => val, + gimli::DW_EH_PE_pcrel => { + // TODO: better handling of sign + let offset = self.len() as u64; + offset.wrapping_sub(val) + } + _ => { + return Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)); + } + }; + self.write_eh_pointer_data(val, eh_pe.format(), size) + } + Address::Symbol { symbol, addend } => match eh_pe.application() { + gimli::DW_EH_PE_pcrel => { + let size = match eh_pe.format() { + gimli::DW_EH_PE_sdata4 => 4, + _ => return Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)), + }; + self.relocs.push(DebugReloc { + offset: self.len() as u32, + size, + name: DebugRelocName::Symbol(symbol), + addend, + kind: object::RelocationKind::Relative, + }); + self.write_udata(0, size) + } + _ => { + return Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)); + } + }, + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs new file mode 100644 index 00000000000..4de84855328 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -0,0 +1,258 @@ +//! Line info generation (`.debug_line`) + +use std::ffi::OsStr; +use std::path::{Component, Path}; + +use crate::prelude::*; + +use rustc_span::{ + FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm, +}; + +use cranelift_codegen::binemit::CodeOffset; +use cranelift_codegen::machinst::MachSrcLoc; + +use gimli::write::{ + Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable, + UnitEntryId, +}; + +// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`. +fn split_path_dir_and_file(path: &Path) -> (&Path, &OsStr) { + let mut iter = path.components(); + let file_name = match iter.next_back() { + Some(Component::Normal(p)) => p, + component => { + panic!( + "Path component {:?} of path {} is an invalid filename", + component, + path.display() + ); + } + }; + let parent = iter.as_path(); + (parent, file_name) +} + +// OPTIMIZATION: Avoid UTF-8 validation on UNIX. +fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + return path.as_bytes(); + } + #[cfg(not(unix))] + { + return path.to_str().unwrap().as_bytes(); + } +} + +pub(crate) const MD5_LEN: usize = 16; + +pub fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> { + if hash.kind == SourceFileHashAlgorithm::Md5 { + let mut buf = [0u8; MD5_LEN]; + buf.copy_from_slice(hash.hash_bytes()); + Some(FileInfo { + timestamp: 0, + size: 0, + md5: buf, + }) + } else { + None + } +} + +fn line_program_add_file( + line_program: &mut LineProgram, + line_strings: &mut LineStringTable, + file: &SourceFile, +) -> FileId { + match &file.name { + FileName::Real(path) => { + let (dir_path, file_name) = split_path_dir_and_file(path.stable_name()); + let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); + let file_name = osstr_as_utf8_bytes(file_name); + + let dir_id = if !dir_name.is_empty() { + let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings); + line_program.add_directory(dir_name) + } else { + line_program.default_directory() + }; + let file_name = LineString::new(file_name, line_program.encoding(), line_strings); + + let info = make_file_info(file.src_hash); + + line_program.file_has_md5 &= info.is_some(); + line_program.add_file(file_name, dir_id, info) + } + // FIXME give more appropriate file names + filename => { + let dir_id = line_program.default_directory(); + let dummy_file_name = LineString::new( + filename.to_string().into_bytes(), + line_program.encoding(), + line_strings, + ); + line_program.add_file(dummy_file_name, dir_id, None) + } + } +} + +impl<'tcx> DebugContext<'tcx> { + pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) { + let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo()); + + let file_id = line_program_add_file( + &mut self.dwarf.unit.line_program, + &mut self.dwarf.line_strings, + &loc.file, + ); + + let entry = self.dwarf.unit.get_mut(entry_id); + + entry.set( + gimli::DW_AT_decl_file, + AttributeValue::FileIndex(Some(file_id)), + ); + entry.set( + gimli::DW_AT_decl_line, + AttributeValue::Udata(loc.line as u64), + ); + // FIXME: probably omit this + entry.set( + gimli::DW_AT_decl_column, + AttributeValue::Udata(loc.col.to_usize() as u64), + ); + } + + pub(super) fn create_debug_lines( + &mut self, + isa: &dyn cranelift_codegen::isa::TargetIsa, + symbol: usize, + entry_id: UnitEntryId, + context: &Context, + function_span: Span, + source_info_set: &indexmap::IndexSet<SourceInfo>, + ) -> CodeOffset { + let tcx = self.tcx; + let line_program = &mut self.dwarf.unit.line_program; + let func = &context.func; + + let line_strings = &mut self.dwarf.line_strings; + let mut last_span = None; + let mut last_file = None; + let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| { + if let Some(last_span) = last_span { + if span == last_span { + line_program.generate_row(); + return; + } + } + last_span = Some(span); + + // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131 + // In order to have a good line stepping behavior in debugger, we overwrite debug + // locations of macro expansions with that of the outermost expansion site + // (unless the crate is being compiled with `-Z debug-macros`). + let span = if !span.from_expansion() || tcx.sess.opts.debugging_opts.debug_macros { + span + } else { + // Walk up the macro expansion chain until we reach a non-expanded span. + // We also stop at the function body level because no line stepping can occur + // at the level above that. + rustc_span::hygiene::walk_chain(span, function_span.ctxt()) + }; + + let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(span.lo()); + + ( + file, + u64::try_from(line).unwrap() + 1, + u64::from((span.lo() - line_pos).to_u32()) + 1, + ) + } + Err(file) => (file, 0, 0), + }; + + // line_program_add_file is very slow. + // Optimize for the common case of the current file not being changed. + let current_file_changed = if let Some(last_file) = &last_file { + // If the allocations are not equal, then the files may still be equal, but that + // is not a problem, as this is just an optimization. + !rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file) + } else { + true + }; + if current_file_changed { + let file_id = line_program_add_file(line_program, line_strings, &file); + line_program.row().file = file_id; + last_file = Some(file.clone()); + } + + line_program.row().line = line; + line_program.row().column = col; + line_program.generate_row(); + }; + + line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 })); + + let mut func_end = 0; + + if let Some(ref mcr) = &context.mach_compile_result { + for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() { + line_program.row().address_offset = u64::from(start); + if !loc.is_default() { + let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap(); + create_row_for_span(line_program, source_info.span); + } else { + create_row_for_span(line_program, function_span); + } + func_end = end; + } + + line_program.end_sequence(u64::from(func_end)); + + func_end = mcr.buffer.total_size(); + } else { + let encinfo = isa.encoding_info(); + let mut blocks = func.layout.blocks().collect::<Vec<_>>(); + blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase + + for block in blocks { + for (offset, inst, size) in func.inst_offsets(block, &encinfo) { + let srcloc = func.srclocs[inst]; + line_program.row().address_offset = u64::from(offset); + if !srcloc.is_default() { + let source_info = + *source_info_set.get_index(srcloc.bits() as usize).unwrap(); + create_row_for_span(line_program, source_info.span); + } else { + create_row_for_span(line_program, function_span); + } + func_end = offset + size; + } + } + line_program.end_sequence(u64::from(func_end)); + } + + assert_ne!(func_end, 0); + + let entry = self.dwarf.unit.get_mut(entry_id); + entry.set( + gimli::DW_AT_low_pc, + AttributeValue::Address(Address::Symbol { symbol, addend: 0 }), + ); + entry.set( + gimli::DW_AT_high_pc, + AttributeValue::Udata(u64::from(func_end)), + ); + + self.emit_location(entry_id, function_span); + + func_end + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs new file mode 100644 index 00000000000..cbf9522b1d7 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -0,0 +1,487 @@ +//! Handling of everything related to debuginfo. + +mod emit; +mod line_info; +mod unwind; + +use crate::prelude::*; + +use rustc_index::vec::IndexVec; + +use cranelift_codegen::entity::EntityRef; +use cranelift_codegen::ir::{StackSlots, ValueLabel, ValueLoc}; +use cranelift_codegen::isa::TargetIsa; +use cranelift_codegen::ValueLocRange; + +use gimli::write::{ + Address, AttributeValue, DwarfUnit, Expression, LineProgram, LineString, Location, + LocationList, Range, RangeList, UnitEntryId, +}; +use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64}; + +pub(crate) use emit::{DebugReloc, DebugRelocName}; +pub(crate) use unwind::UnwindContext; + +fn target_endian(tcx: TyCtxt<'_>) -> RunTimeEndian { + use rustc_target::abi::Endian; + + match tcx.data_layout.endian { + Endian::Big => RunTimeEndian::Big, + Endian::Little => RunTimeEndian::Little, + } +} + +pub(crate) struct DebugContext<'tcx> { + tcx: TyCtxt<'tcx>, + + endian: RunTimeEndian, + + dwarf: DwarfUnit, + unit_range_list: RangeList, + + clif_types: FxHashMap<Type, UnitEntryId>, + types: FxHashMap<Ty<'tcx>, UnitEntryId>, +} + +impl<'tcx> DebugContext<'tcx> { + pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { + let encoding = Encoding { + format: Format::Dwarf32, + // TODO: this should be configurable + // macOS doesn't seem to support DWARF > 3 + // 5 version is required for md5 file hash + version: if tcx.sess.target.options.is_like_osx { + 3 + } else { + // FIXME change to version 5 once the gdb and lldb shipping with the latest debian + // support it. + 4 + }, + address_size: isa.frontend_config().pointer_bytes(), + }; + + let mut dwarf = DwarfUnit::new(encoding); + + // FIXME: how to get version when building out of tree? + // Normally this would use option_env!("CFG_VERSION"). + let producer = format!("cg_clif (rustc {})", "unknown version"); + let comp_dir = tcx.sess.working_dir.0.to_string_lossy().into_owned(); + let (name, file_info) = match tcx.sess.local_crate_source_file.clone() { + Some(path) => { + let name = path.to_string_lossy().into_owned(); + (name, None) + } + None => (tcx.crate_name(LOCAL_CRATE).to_string(), None), + }; + + let mut line_program = LineProgram::new( + encoding, + LineEncoding::default(), + LineString::new(comp_dir.as_bytes(), encoding, &mut dwarf.line_strings), + LineString::new(name.as_bytes(), encoding, &mut dwarf.line_strings), + file_info, + ); + line_program.file_has_md5 = file_info.is_some(); + + dwarf.unit.line_program = line_program; + + { + let name = dwarf.strings.add(name); + let comp_dir = dwarf.strings.add(comp_dir); + + let root = dwarf.unit.root(); + let root = dwarf.unit.get_mut(root); + root.set( + gimli::DW_AT_producer, + AttributeValue::StringRef(dwarf.strings.add(producer)), + ); + root.set( + gimli::DW_AT_language, + AttributeValue::Language(gimli::DW_LANG_Rust), + ); + root.set(gimli::DW_AT_name, AttributeValue::StringRef(name)); + root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir)); + root.set( + gimli::DW_AT_low_pc, + AttributeValue::Address(Address::Constant(0)), + ); + } + + DebugContext { + tcx, + + endian: target_endian(tcx), + + dwarf, + unit_range_list: RangeList(Vec::new()), + + clif_types: FxHashMap::default(), + types: FxHashMap::default(), + } + } + + fn dwarf_ty_for_clif_ty(&mut self, ty: Type) -> UnitEntryId { + if let Some(type_id) = self.clif_types.get(&ty) { + return *type_id; + } + + let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag); + + let primitive = |dwarf: &mut DwarfUnit, ate| { + let type_id = new_entry(dwarf, gimli::DW_TAG_base_type); + let type_entry = dwarf.unit.get_mut(type_id); + type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate)); + type_id + }; + + let type_id = if ty.is_bool() { + primitive(&mut self.dwarf, gimli::DW_ATE_boolean) + } else if ty.is_int() { + primitive(&mut self.dwarf, gimli::DW_ATE_address) + } else if ty.is_float() { + primitive(&mut self.dwarf, gimli::DW_ATE_float) + } else { + new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type) + }; + + let type_entry = self.dwarf.unit.get_mut(type_id); + type_entry.set( + gimli::DW_AT_name, + AttributeValue::String(format!("{}", ty).replace('i', "u").into_bytes()), + ); + type_entry.set( + gimli::DW_AT_byte_size, + AttributeValue::Udata(u64::from(ty.bytes())), + ); + + type_id + } + + fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId { + if let Some(type_id) = self.types.get(ty) { + return *type_id; + } + + let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag); + + let primitive = |dwarf: &mut DwarfUnit, ate| { + let type_id = new_entry(dwarf, gimli::DW_TAG_base_type); + let type_entry = dwarf.unit.get_mut(type_id); + type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate)); + type_id + }; + + let name = format!("{}", ty); + let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap(); + + let type_id = match ty.kind() { + ty::Bool => primitive(&mut self.dwarf, gimli::DW_ATE_boolean), + ty::Char => primitive(&mut self.dwarf, gimli::DW_ATE_UTF), + ty::Uint(_) => primitive(&mut self.dwarf, gimli::DW_ATE_unsigned), + ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed), + ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float), + ty::Ref(_, pointee_ty, _mutbl) + | ty::RawPtr(ty::TypeAndMut { + ty: pointee_ty, + mutbl: _mutbl, + }) => { + let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type); + + // Ensure that type is inserted before recursing to avoid duplicates + self.types.insert(ty, type_id); + + let pointee = self.dwarf_ty(pointee_ty); + + let type_entry = self.dwarf.unit.get_mut(type_id); + + //type_entry.set(gimli::DW_AT_mutable, AttributeValue::Flag(mutbl == rustc_hir::Mutability::Mut)); + type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee)); + + type_id + } + ty::Adt(adt_def, _substs) if adt_def.is_struct() && !layout.is_unsized() => { + let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type); + + // Ensure that type is inserted before recursing to avoid duplicates + self.types.insert(ty, type_id); + + let variant = adt_def.non_enum_variant(); + + for (field_idx, field_def) in variant.fields.iter().enumerate() { + let field_offset = layout.fields.offset(field_idx); + let field_layout = layout + .field( + &layout::LayoutCx { + tcx: self.tcx, + param_env: ParamEnv::reveal_all(), + }, + field_idx, + ) + .unwrap(); + + let field_type = self.dwarf_ty(field_layout.ty); + + let field_id = self.dwarf.unit.add(type_id, gimli::DW_TAG_member); + let field_entry = self.dwarf.unit.get_mut(field_id); + + field_entry.set( + gimli::DW_AT_name, + AttributeValue::String(field_def.ident.as_str().to_string().into_bytes()), + ); + field_entry.set( + gimli::DW_AT_data_member_location, + AttributeValue::Udata(field_offset.bytes()), + ); + field_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(field_type)); + } + + type_id + } + _ => new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type), + }; + + let type_entry = self.dwarf.unit.get_mut(type_id); + + type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes())); + type_entry.set( + gimli::DW_AT_byte_size, + AttributeValue::Udata(layout.size.bytes()), + ); + + self.types.insert(ty, type_id); + + type_id + } + + fn define_local(&mut self, scope: UnitEntryId, name: String, ty: Ty<'tcx>) -> UnitEntryId { + let dw_ty = self.dwarf_ty(ty); + + let var_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable); + let var_entry = self.dwarf.unit.get_mut(var_id); + + var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes())); + var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty)); + + var_id + } + + pub(crate) fn define_function( + &mut self, + instance: Instance<'tcx>, + func_id: FuncId, + name: &str, + isa: &dyn TargetIsa, + context: &Context, + source_info_set: &indexmap::IndexSet<SourceInfo>, + local_map: IndexVec<mir::Local, CPlace<'tcx>>, + ) { + let symbol = func_id.as_u32() as usize; + let mir = self.tcx.instance_mir(instance.def); + + // FIXME: add to appropriate scope instead of root + let scope = self.dwarf.unit.root(); + + let entry_id = self.dwarf.unit.add(scope, gimli::DW_TAG_subprogram); + let entry = self.dwarf.unit.get_mut(entry_id); + let name_id = self.dwarf.strings.add(name); + // Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped. + entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id)); + entry.set( + gimli::DW_AT_linkage_name, + AttributeValue::StringRef(name_id), + ); + + let end = + self.create_debug_lines(isa, symbol, entry_id, context, mir.span, source_info_set); + + self.unit_range_list.0.push(Range::StartLength { + begin: Address::Symbol { symbol, addend: 0 }, + length: u64::from(end), + }); + + if isa.get_mach_backend().is_some() { + return; // Not yet implemented for the AArch64 backend. + } + + let func_entry = self.dwarf.unit.get_mut(entry_id); + // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped. + func_entry.set( + gimli::DW_AT_low_pc, + AttributeValue::Address(Address::Symbol { symbol, addend: 0 }), + ); + // Using Udata for DW_AT_high_pc requires at least DWARF4 + func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end))); + + // FIXME Remove once actual debuginfo for locals works. + for (i, (param, &val)) in context + .func + .signature + .params + .iter() + .zip( + context + .func + .dfg + .block_params(context.func.layout.entry_block().unwrap()), + ) + .enumerate() + { + use cranelift_codegen::ir::ArgumentPurpose; + let base_name = match param.purpose { + ArgumentPurpose::Normal => "arg", + ArgumentPurpose::StructArgument(_) => "struct_arg", + ArgumentPurpose::StructReturn => "sret", + ArgumentPurpose::Link + | ArgumentPurpose::FramePointer + | ArgumentPurpose::CalleeSaved => continue, + ArgumentPurpose::VMContext + | ArgumentPurpose::SignatureId + | ArgumentPurpose::CallerTLS + | ArgumentPurpose::CalleeTLS + | ArgumentPurpose::StackLimit => unreachable!(), + }; + let name = format!("{}{}", base_name, i); + + let dw_ty = self.dwarf_ty_for_clif_ty(param.value_type); + let loc = + translate_loc(isa, context.func.locations[val], &context.func.stack_slots).unwrap(); + + let arg_id = self + .dwarf + .unit + .add(entry_id, gimli::DW_TAG_formal_parameter); + let var_entry = self.dwarf.unit.get_mut(arg_id); + + var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes())); + var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty)); + var_entry.set(gimli::DW_AT_location, AttributeValue::Exprloc(loc)); + } + + // FIXME make it more reliable and implement scopes before re-enabling this. + if false { + let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap(); + + for (local, _local_decl) in mir.local_decls.iter_enumerated() { + let ty = self.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + &mir.local_decls[local].ty, + ); + let var_id = self.define_local(entry_id, format!("{:?}", local), ty); + + let location = place_location( + self, + isa, + symbol, + context, + &local_map, + &value_labels_ranges, + Place { + local, + projection: ty::List::empty(), + }, + ); + + let var_entry = self.dwarf.unit.get_mut(var_id); + var_entry.set(gimli::DW_AT_location, location); + } + } + + // FIXME create locals for all entries in mir.var_debug_info + } +} + +fn place_location<'tcx>( + debug_context: &mut DebugContext<'tcx>, + isa: &dyn TargetIsa, + symbol: usize, + context: &Context, + local_map: &IndexVec<mir::Local, CPlace<'tcx>>, + #[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap< + ValueLabel, + Vec<ValueLocRange>, + >, + place: Place<'tcx>, +) -> AttributeValue { + assert!(place.projection.is_empty()); // FIXME implement them + + match local_map[place.local].inner() { + CPlaceInner::Var(_local, var) => { + let value_label = cranelift_codegen::ir::ValueLabel::new(var.index()); + if let Some(value_loc_ranges) = value_labels_ranges.get(&value_label) { + let loc_list = LocationList( + value_loc_ranges + .iter() + .map(|value_loc_range| Location::StartEnd { + begin: Address::Symbol { + symbol, + addend: i64::from(value_loc_range.start), + }, + end: Address::Symbol { + symbol, + addend: i64::from(value_loc_range.end), + }, + data: translate_loc( + isa, + value_loc_range.loc, + &context.func.stack_slots, + ) + .unwrap(), + }) + .collect(), + ); + let loc_list_id = debug_context.dwarf.unit.locations.add(loc_list); + + AttributeValue::LocationListRef(loc_list_id) + } else { + // FIXME set value labels for unused locals + + AttributeValue::Exprloc(Expression::new()) + } + } + CPlaceInner::VarPair(_, _, _) => { + // FIXME implement this + + AttributeValue::Exprloc(Expression::new()) + } + CPlaceInner::VarLane(_, _, _) => { + // FIXME implement this + + AttributeValue::Exprloc(Expression::new()) + } + CPlaceInner::Addr(_, _) => { + // FIXME implement this (used by arguments and returns) + + AttributeValue::Exprloc(Expression::new()) + + // For PointerBase::Stack: + //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot), &context.func.stack_slots).unwrap()) + } + } +} + +// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137 +fn translate_loc( + isa: &dyn TargetIsa, + loc: ValueLoc, + stack_slots: &StackSlots, +) -> Option<Expression> { + match loc { + ValueLoc::Reg(reg) => { + let machine_reg = isa.map_dwarf_register(reg).unwrap(); + let mut expr = Expression::new(); + expr.op_reg(gimli::Register(machine_reg)); + Some(expr) + } + ValueLoc::Stack(ss) => { + if let Some(ss_offset) = stack_slots[ss].offset { + let mut expr = Expression::new(); + expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16); + Some(expr) + } else { + None + } + } + _ => None, + } +} diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs new file mode 100644 index 00000000000..61ebd931d2f --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -0,0 +1,167 @@ +//! Unwind info generation (`.eh_frame`) + +use crate::prelude::*; + +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; + +use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; + +use crate::backend::WriteDebugInfo; + +pub(crate) struct UnwindContext<'tcx> { + tcx: TyCtxt<'tcx>, + frame_table: FrameTable, + cie_id: Option<CieId>, +} + +impl<'tcx> UnwindContext<'tcx> { + pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { + let mut frame_table = FrameTable::default(); + + let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { + if isa.flags().is_pic() { + cie.fde_address_encoding = + gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); + } + Some(frame_table.add_cie(cie)) + } else { + None + }; + + UnwindContext { + tcx, + frame_table, + cie_id, + } + } + + pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { + let unwind_info = if let Some(unwind_info) = context.create_unwind_info(isa).unwrap() { + unwind_info + } else { + return; + }; + + match unwind_info { + UnwindInfo::SystemV(unwind_info) => { + self.frame_table.add_fde( + self.cie_id.unwrap(), + unwind_info.to_fde(Address::Symbol { + symbol: func_id.as_u32() as usize, + addend: 0, + }), + ); + } + UnwindInfo::WindowsX64(_) => { + // FIXME implement this + } + } + } + + pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( + self.tcx, + ))); + self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); + + if !eh_frame.0.writer.slice().is_empty() { + let id = eh_frame.id(); + let section_id = product.add_debug_section(id, eh_frame.0.writer.into_vec()); + let mut section_map = FxHashMap::default(); + section_map.insert(id, section_id); + + for reloc in &eh_frame.0.relocs { + product.add_debug_reloc(§ion_map, §ion_id, reloc); + } + } + } + + #[cfg(feature = "jit")] + pub(crate) unsafe fn register_jit( + self, + jit_product: &cranelift_simplejit::SimpleJITProduct, + ) -> Option<UnwindRegistry> { + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian( + self.tcx, + ))); + self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); + + if eh_frame.0.writer.slice().is_empty() { + return None; + } + + let mut eh_frame = eh_frame.0.relocate_for_jit(jit_product); + + // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. + eh_frame.extend(&[0, 0, 0, 0]); + + let mut registrations = Vec::new(); + + // ======================================================================= + // Everything after this line up to the end of the file is loosly based on + // https://github.com/bytecodealliance/wasmtime/blob/4471a82b0c540ff48960eca6757ccce5b1b5c3e4/crates/jit/src/unwind/systemv.rs + #[cfg(target_os = "macos")] + { + // On macOS, `__register_frame` takes a pointer to a single FDE + let start = eh_frame.as_ptr(); + let end = start.add(eh_frame.len()); + let mut current = start; + + // Walk all of the entries in the frame table and register them + while current < end { + let len = std::ptr::read::<u32>(current as *const u32) as usize; + + // Skip over the CIE + if current != start { + __register_frame(current); + registrations.push(current as usize); + } + + // Move to the next table entry (+4 because the length itself is not inclusive) + current = current.add(len + 4); + } + } + #[cfg(not(target_os = "macos"))] + { + // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 + let ptr = eh_frame.as_ptr(); + __register_frame(ptr); + registrations.push(ptr as usize); + } + + Some(UnwindRegistry { + _frame_table: eh_frame, + registrations, + }) + } +} + +/// Represents a registry of function unwind information for System V ABI. +pub(crate) struct UnwindRegistry { + _frame_table: Vec<u8>, + registrations: Vec<usize>, +} + +extern "C" { + // libunwind import + fn __register_frame(fde: *const u8); + fn __deregister_frame(fde: *const u8); +} + +impl Drop for UnwindRegistry { + fn drop(&mut self) { + unsafe { + // libgcc stores the frame entries as a linked list in decreasing sort order + // based on the PC value of the registered entry. + // + // As we store the registrations in increasing order, it would be O(N^2) to + // deregister in that order. + // + // To ensure that we just pop off the first element in the list upon every + // deregistration, walk our list of registrations backwards. + for fde in self.registrations.iter().rev() { + __deregister_frame(*fde as *const _); + } + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs new file mode 100644 index 00000000000..d15bc36ad05 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs @@ -0,0 +1,170 @@ +//! Handling of enum discriminants +//! +//! Adapted from https://github.com/rust-lang/rust/blob/d760df5aea483aae041c9a241e7acacf48f75035/src/librustc_codegen_ssa/mir/place.rs + +use rustc_target::abi::{Int, TagEncoding, Variants}; + +use crate::prelude::*; + +pub(crate) fn codegen_set_discriminant<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + place: CPlace<'tcx>, + variant_index: VariantIdx, +) { + let layout = place.layout(); + if layout.for_variant(fx, variant_index).abi.is_uninhabited() { + return; + } + match layout.variants { + Variants::Single { index } => { + assert_eq!(index, variant_index); + } + Variants::Multiple { + tag: _, + tag_field, + tag_encoding: TagEncoding::Direct, + variants: _, + } => { + let ptr = place.place_field(fx, mir::Field::new(tag_field)); + let to = layout + .ty + .discriminant_for_variant(fx.tcx, variant_index) + .unwrap() + .val; + let discr = CValue::const_val(fx, ptr.layout(), to); + ptr.write_cvalue(fx, discr); + } + Variants::Multiple { + tag: _, + tag_field, + tag_encoding: + TagEncoding::Niche { + dataful_variant, + ref niche_variants, + niche_start, + }, + variants: _, + } => { + if variant_index != dataful_variant { + let niche = place.place_field(fx, mir::Field::new(tag_field)); + let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); + let niche_value = u128::from(niche_value).wrapping_add(niche_start); + let niche_llval = CValue::const_val(fx, niche.layout(), niche_value); + niche.write_cvalue(fx, niche_llval); + } + } + } +} + +pub(crate) fn codegen_get_discriminant<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + value: CValue<'tcx>, + dest_layout: TyAndLayout<'tcx>, +) -> CValue<'tcx> { + let layout = value.layout(); + + if layout.abi == Abi::Uninhabited { + return trap_unreachable_ret_value( + fx, + dest_layout, + "[panic] Tried to get discriminant for uninhabited type.", + ); + } + + let (tag_scalar, tag_field, tag_encoding) = match &layout.variants { + Variants::Single { index } => { + let discr_val = layout + .ty + .discriminant_for_variant(fx.tcx, *index) + .map_or(u128::from(index.as_u32()), |discr| discr.val); + return CValue::const_val(fx, dest_layout, discr_val); + } + Variants::Multiple { + tag, + tag_field, + tag_encoding, + variants: _, + } => (tag, *tag_field, tag_encoding), + }; + + let cast_to = fx.clif_type(dest_layout.ty).unwrap(); + + // Read the tag/niche-encoded discriminant from memory. + let tag = value.value_field(fx, mir::Field::new(tag_field)); + let tag = tag.load_scalar(fx); + + // Decode the discriminant (specifically if it's niche-encoded). + match *tag_encoding { + TagEncoding::Direct => { + let signed = match tag_scalar.value { + Int(_, signed) => signed, + _ => false, + }; + let val = clif_intcast(fx, tag, cast_to, signed); + CValue::by_val(val, dest_layout) + } + TagEncoding::Niche { + dataful_variant, + ref niche_variants, + niche_start, + } => { + // Rebase from niche values to discriminants, and check + // whether the result is in range for the niche variants. + + // We first compute the "relative discriminant" (wrt `niche_variants`), + // that is, if `n = niche_variants.end() - niche_variants.start()`, + // we remap `niche_start..=niche_start + n` (which may wrap around) + // to (non-wrap-around) `0..=n`, to be able to check whether the + // discriminant corresponds to a niche variant with one comparison. + // We also can't go directly to the (variant index) discriminant + // and check that it is in the range `niche_variants`, because + // that might not fit in the same type, on top of needing an extra + // comparison (see also the comment on `let niche_discr`). + let relative_discr = if niche_start == 0 { + tag + } else { + // FIXME handle niche_start > i64::MAX + fx.bcx + .ins() + .iadd_imm(tag, -i64::try_from(niche_start).unwrap()) + }; + let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); + let is_niche = { + codegen_icmp_imm( + fx, + IntCC::UnsignedLessThanOrEqual, + relative_discr, + i128::from(relative_max), + ) + }; + + // NOTE(eddyb) this addition needs to be performed on the final + // type, in case the niche itself can't represent all variant + // indices (e.g. `u8` niche with more than `256` variants, + // but enough uninhabited variants so that the remaining variants + // fit in the niche). + // In other words, `niche_variants.end - niche_variants.start` + // is representable in the niche, but `niche_variants.end` + // might not be, in extreme cases. + let niche_discr = { + let relative_discr = if relative_max == 0 { + // HACK(eddyb) since we have only one niche, we know which + // one it is, and we can avoid having a dynamic value here. + fx.bcx.ins().iconst(cast_to, 0) + } else { + clif_intcast(fx, relative_discr, cast_to, false) + }; + fx.bcx + .ins() + .iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32())) + }; + + let dataful_variant = fx + .bcx + .ins() + .iconst(cast_to, i64::from(dataful_variant.as_u32())); + let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant); + CValue::by_val(discr, dest_layout) + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs new file mode 100644 index 00000000000..ff0b994c9a9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -0,0 +1,450 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::path::PathBuf; + +use rustc_codegen_ssa::back::linker::LinkerInfo; +use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::middle::cstore::EncodedMetadata; +use rustc_middle::mir::mono::CodegenUnit; +use rustc_session::cgu_reuse_tracker::CguReuse; +use rustc_session::config::{DebugInfo, OutputType}; + +use cranelift_object::{ObjectModule, ObjectProduct}; + +use crate::prelude::*; + +use crate::backend::AddConstructor; + +fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule { + let module = crate::backend::make_module(tcx.sess, name); + assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); + module +} + +struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); + +impl<HCX> HashStable<HCX> for ModuleCodegenResult { + fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { + // do nothing + } +} + +fn emit_module( + tcx: TyCtxt<'_>, + name: String, + kind: ModuleKind, + module: ObjectModule, + debug: Option<DebugContext<'_>>, + unwind_context: UnwindContext<'_>, + map_product: impl FnOnce(ObjectProduct) -> ObjectProduct, +) -> ModuleCodegenResult { + let mut product = module.finish(); + + if let Some(mut debug) = debug { + debug.emit(&mut product); + } + + unwind_context.emit(&mut product); + + let product = map_product(product); + + let tmp_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, Some(&name)); + let obj = product.object.write().unwrap(); + if let Err(err) = std::fs::write(&tmp_file, obj) { + tcx.sess + .fatal(&format!("error writing object file: {}", err)); + } + + let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { + None + } else { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + tcx.sess, + &name, + &Some(tmp_file.clone()), + ) + }; + + ModuleCodegenResult( + CompiledModule { + name, + kind, + object: Some(tmp_file), + bytecode: None, + }, + work_product, + ) +} + +fn reuse_workproduct_for_cgu( + tcx: TyCtxt<'_>, + cgu: &CodegenUnit<'_>, + work_products: &mut FxHashMap<WorkProductId, WorkProduct>, +) -> CompiledModule { + let incr_comp_session_dir = tcx.sess.incr_comp_session_dir(); + let mut object = None; + let work_product = cgu.work_product(tcx); + if let Some(saved_file) = &work_product.saved_file { + let obj_out = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, Some(&cgu.name().as_str())); + object = Some(obj_out.clone()); + let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, &saved_file); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { + tcx.sess.err(&format!( + "unable to copy {} to {}: {}", + source_file.display(), + obj_out.display(), + err + )); + } + } + + work_products.insert(cgu.work_product_id(), work_product); + + CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object, + bytecode: None, + } +} + +fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodegenResult { + let cgu = tcx.codegen_unit(cgu_name); + let mono_items = cgu.items_in_deterministic_order(tcx); + + let mut module = new_module(tcx, cgu_name.as_str().to_string()); + + // Initialize the global atomic mutex using a constructor for proc-macros. + // FIXME implement atomic instructions in Cranelift. + let mut init_atomics_mutex_from_constructor = None; + if tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::ProcMacro) + { + if mono_items.iter().any(|(mono_item, _)| match mono_item { + rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx + .symbol_name(Instance::mono(tcx, *def_id)) + .name + .contains("__rustc_proc_macro_decls_"), + _ => false, + }) { + init_atomics_mutex_from_constructor = + Some(crate::atomic_shim::init_global_lock_constructor( + &mut module, + &format!("{}_init_atomics_mutex", cgu_name.as_str()), + )); + } + } + + let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None); + super::codegen_mono_items(&mut cx, mono_items); + let (mut module, global_asm, debug, mut unwind_context) = + tcx.sess.time("finalize CodegenCx", || cx.finalize()); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false); + + let codegen_result = emit_module( + tcx, + cgu.name().as_str().to_string(), + ModuleKind::Regular, + module, + debug, + unwind_context, + |mut product| { + if let Some(func_id) = init_atomics_mutex_from_constructor { + product.add_constructor(func_id); + } + + product + }, + ); + + codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + + codegen_result +} + +pub(super) fn run_aot( + tcx: TyCtxt<'_>, + metadata: EncodedMetadata, + need_metadata_module: bool, +) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> { + let mut work_products = FxHashMap::default(); + + let cgus = if tcx.sess.opts.output_types.should_codegen() { + tcx.collect_and_partition_mono_items(LOCAL_CRATE).1 + } else { + // If only `--emit metadata` is used, we shouldn't perform any codegen. + // Also `tcx.collect_and_partition_mono_items` may panic in that case. + &[] + }; + + if tcx.dep_graph.is_fully_enabled() { + for cgu in &*cgus { + tcx.ensure().codegen_unit(cgu.name()); + } + } + + let modules = super::time(tcx, "codegen mono items", || { + cgus.iter() + .map(|cgu| { + let cgu_reuse = determine_cgu_reuse(tcx, cgu); + tcx.sess + .cgu_reuse_tracker + .set_actual_reuse(&cgu.name().as_str(), cgu_reuse); + + match cgu_reuse { + _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} + CguReuse::No => {} + CguReuse::PreLto => { + return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); + } + CguReuse::PostLto => unreachable!(), + } + + let dep_node = cgu.codegen_dep_node(tcx); + let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( + dep_node, + tcx, + cgu.name(), + module_codegen, + rustc_middle::dep_graph::hash_result, + ); + + if let Some((id, product)) = work_product { + work_products.insert(id, product); + } + + module + }) + .collect::<Vec<_>>() + }); + + tcx.sess.abort_if_errors(); + + let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); + let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa()); + let created_alloc_shim = + crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); + + let allocator_module = if created_alloc_shim { + let ModuleCodegenResult(module, work_product) = emit_module( + tcx, + "allocator_shim".to_string(), + ModuleKind::Allocator, + allocator_module, + None, + allocator_unwind_context, + |product| product, + ); + if let Some((id, product)) = work_product { + work_products.insert(id, product); + } + Some(module) + } else { + None + }; + + rustc_incremental::assert_dep_graph(tcx); + rustc_incremental::save_dep_graph(tcx); + + let metadata_module = if need_metadata_module { + let _timer = tcx.prof.generic_activity("codegen crate metadata"); + let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || { + use rustc_middle::mir::mono::CodegenUnitNameBuilder; + + let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); + let metadata_cgu_name = cgu_name_builder + .build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")) + .as_str() + .to_string(); + + let tmp_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Metadata, Some(&metadata_cgu_name)); + + let obj = crate::backend::with_object(tcx.sess, &metadata_cgu_name, |object| { + crate::metadata::write_metadata(tcx, object); + }); + + if let Err(err) = std::fs::write(&tmp_file, obj) { + tcx.sess + .fatal(&format!("error writing metadata object file: {}", err)); + } + + (metadata_cgu_name, tmp_file) + }); + + Some(CompiledModule { + name: metadata_cgu_name, + kind: ModuleKind::Metadata, + object: Some(tmp_file), + bytecode: None, + }) + } else { + None + }; + + if tcx.sess.opts.output_types.should_codegen() { + rustc_incremental::assert_module_sources::assert_module_sources(tcx); + } + + Box::new(( + CodegenResults { + crate_name: tcx.crate_name(LOCAL_CRATE), + modules, + allocator_module, + metadata_module, + metadata, + windows_subsystem: None, // Windows is not yet supported + linker_info: LinkerInfo::new(tcx), + crate_info: CrateInfo::new(tcx), + }, + work_products, + )) +} + +fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { + use std::io::Write; + use std::process::{Command, Stdio}; + + if global_asm.is_empty() { + return; + } + + if cfg!(not(feature = "inline_asm")) + || tcx.sess.target.options.is_like_osx + || tcx.sess.target.options.is_like_windows + { + if global_asm.contains("__rust_probestack") { + return; + } + + // FIXME fix linker error on macOS + if cfg!(not(feature = "inline_asm")) { + tcx.sess.fatal( + "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", + ); + } else { + tcx.sess + .fatal("asm! and global_asm! are not yet supported on macOS and Windows"); + } + } + + let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); + let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); + + // Remove all LLVM style comments + let global_asm = global_asm + .lines() + .map(|line| { + if let Some(index) = line.find("//") { + &line[0..index] + } else { + line + } + }) + .collect::<Vec<_>>() + .join("\n"); + + let output_object_file = tcx + .output_filenames(LOCAL_CRATE) + .temp_path(OutputType::Object, Some(cgu_name)); + + // Assemble `global_asm` + let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); + let mut child = Command::new(assembler) + .arg("-o") + .arg(&global_asm_object_file) + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + child + .stdin + .take() + .unwrap() + .write_all(global_asm.as_bytes()) + .unwrap(); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + tcx.sess + .fatal(&format!("Failed to assemble `{}`", global_asm)); + } + + // Link the global asm and main object file together + let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); + std::fs::rename(&output_object_file, &main_object_file).unwrap(); + let status = Command::new(linker) + .arg("-r") // Create a new object file + .arg("-o") + .arg(output_object_file) + .arg(&main_object_file) + .arg(&global_asm_object_file) + .status() + .unwrap(); + if !status.success() { + tcx.sess.fatal(&format!( + "Failed to link `{}` and `{}` together", + main_object_file.display(), + global_asm_object_file.display(), + )); + } + + std::fs::remove_file(global_asm_object_file).unwrap(); + std::fs::remove_file(main_object_file).unwrap(); +} + +fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { + let mut new_filename = path.file_stem().unwrap().to_owned(); + new_filename.push(postfix); + if let Some(extension) = path.extension() { + new_filename.push("."); + new_filename.push(extension); + } + path.set_file_name(new_filename); + path +} + +// Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953 +fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse { + if !tcx.dep_graph.is_fully_enabled() { + return CguReuse::No; + } + + let work_product_id = &cgu.work_product_id(); + if tcx + .dep_graph + .previous_work_product(work_product_id) + .is_none() + { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. + return CguReuse::No; + } + + // Try to mark the CGU as green. If it we can do so, it means that nothing + // affecting the LLVM module has changed and we can re-use a cached version. + // If we compile with any kind of LTO, this means we can re-use the bitcode + // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only + // know that later). If we are not doing LTO, there is only one optimized + // version of each module, so we re-use that. + let dep_node = cgu.codegen_dep_node(tcx); + assert!( + !tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name() + ); + + if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + CguReuse::PreLto + } else { + CguReuse::No + } +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs new file mode 100644 index 00000000000..b5bab3d9e1e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -0,0 +1,169 @@ +//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! files. + +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; + +use rustc_codegen_ssa::CrateInfo; + +use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule}; + +use crate::prelude::*; + +pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! { + if !tcx.sess.opts.output_types.should_codegen() { + tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); + } + + #[cfg(unix)] + unsafe { + // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so + // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again + // as global. + // FIXME remove once atomic_shim is gone + + let mut dl_info: libc::Dl_info = std::mem::zeroed(); + assert_ne!( + libc::dladdr(run_jit as *const libc::c_void, &mut dl_info), + 0 + ); + assert_ne!( + libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL), + std::ptr::null_mut(), + ); + } + + let imported_symbols = load_imported_symbols_for_jit(tcx); + + let mut jit_builder = SimpleJITBuilder::with_isa( + crate::build_isa(tcx.sess, false), + cranelift_module::default_libcall_names(), + ); + jit_builder.symbols(imported_symbols); + let mut jit_module = SimpleJITModule::new(jit_builder); + assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); + + let sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new( + jit_module.target_config().pointer_type(), /*isize*/ + )], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let main_func_id = jit_module + .declare_function("main", Linkage::Import, &sig) + .unwrap(); + + let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let mono_items = cgus + .iter() + .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter()) + .flatten() + .collect::<FxHashMap<_, (_, _)>>() + .into_iter() + .collect::<Vec<(_, (_, _))>>(); + + let mut cx = crate::CodegenCx::new(tcx, jit_module, false); + + let (mut jit_module, global_asm, _debug, mut unwind_context) = + super::time(tcx, "codegen mono items", || { + super::codegen_mono_items(&mut cx, mono_items); + tcx.sess.time("finalize CodegenCx", || cx.finalize()) + }); + if !global_asm.is_empty() { + tcx.sess.fatal("Global asm is not supported in JIT mode"); + } + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true); + crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); + + tcx.sess.abort_if_errors(); + + let jit_product = jit_module.finish(); + + let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_product) }; + + let finalized_main: *const u8 = jit_product.lookup_func(main_func_id); + + println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed"); + + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_main) }; + + let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); + let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) + .chain(args.split(" ")) + .map(|arg| CString::new(arg).unwrap()) + .collect::<Vec<_>>(); + let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>(); + + // Push a null pointer as a terminating argument. This is required by POSIX and + // useful as some dynamic linkers use it as a marker to jump over. + argv.push(std::ptr::null()); + + let ret = f(args.len() as c_int, argv.as_ptr()); + + std::process::exit(ret); +} + +fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { + use rustc_middle::middle::dependency_format::Linkage; + + let mut dylib_paths = Vec::new(); + + let crate_info = CrateInfo::new(tcx); + let formats = tcx.dependency_formats(LOCAL_CRATE); + let data = &formats + .iter() + .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable) + .unwrap() + .1; + for &(cnum, _) in &crate_info.used_crates_dynamic { + let src = &crate_info.used_crate_source[&cnum]; + match data[cnum.as_usize() - 1] { + Linkage::NotLinked | Linkage::IncludedFromDylib => {} + Linkage::Static => { + let name = tcx.crate_name(cnum); + let mut err = tcx + .sess + .struct_err(&format!("Can't load static lib {}", name.as_str())); + err.note("rustc_codegen_cranelift can only load dylibs in JIT mode."); + err.emit(); + } + Linkage::Dynamic => { + dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); + } + } + } + + let mut imported_symbols = Vec::new(); + for path in dylib_paths { + use object::Object; + let lib = libloading::Library::new(&path).unwrap(); + let obj = std::fs::read(path).unwrap(); + let obj = object::File::parse(&obj).unwrap(); + imported_symbols.extend(obj.dynamic_symbols().filter_map(|(_idx, symbol)| { + let name = symbol.name().unwrap().to_string(); + if name.is_empty() || !symbol.is_global() || symbol.is_undefined() { + return None; + } + let dlsym_name = if cfg!(target_os = "macos") { + // On macOS `dlsym` expects the name without leading `_`. + assert!(name.starts_with("_"), "{:?}", name); + &name[1..] + } else { + &name + }; + let symbol: libloading::Symbol<'_, *const u8> = + unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap(); + Some((name, *symbol)) + })); + std::mem::forget(lib) + } + + tcx.sess.abort_if_errors(); + + imported_symbols +} diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs new file mode 100644 index 00000000000..2fb353ca162 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -0,0 +1,120 @@ +//! Drivers are responsible for calling [`codegen_mono_items`] and performing any further actions +//! like JIT executing or writing object files. + +use std::any::Any; + +use rustc_middle::middle::cstore::EncodedMetadata; +use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; + +use crate::prelude::*; + +mod aot; +#[cfg(feature = "jit")] +mod jit; + +pub(crate) fn codegen_crate( + tcx: TyCtxt<'_>, + metadata: EncodedMetadata, + need_metadata_module: bool, + config: crate::BackendConfig, +) -> Box<dyn Any> { + tcx.sess.abort_if_errors(); + + if config.use_jit { + let is_executable = tcx + .sess + .crate_types() + .contains(&rustc_session::config::CrateType::Executable); + if !is_executable { + tcx.sess.fatal("can't jit non-executable crate"); + } + + #[cfg(feature = "jit")] + let _: ! = jit::run_jit(tcx); + + #[cfg(not(feature = "jit"))] + tcx.sess + .fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + + aot::run_aot(tcx, metadata, need_metadata_module) +} + +fn codegen_mono_items<'tcx>( + cx: &mut crate::CodegenCx<'tcx, impl Module>, + mono_items: Vec<(MonoItem<'tcx>, (RLinkage, Visibility))>, +) { + cx.tcx.sess.time("predefine functions", || { + for &(mono_item, (linkage, visibility)) in &mono_items { + match mono_item { + MonoItem::Fn(instance) => { + let (name, sig) = get_function_name_and_sig( + cx.tcx, + cx.module.isa().triple(), + instance, + false, + ); + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + cx.module.declare_function(&name, linkage, &sig).unwrap(); + } + MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} + } + } + }); + + for (mono_item, (linkage, visibility)) in mono_items { + let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + trans_mono_item(cx, mono_item, linkage); + } +} + +fn trans_mono_item<'tcx, M: Module>( + cx: &mut crate::CodegenCx<'tcx, M>, + mono_item: MonoItem<'tcx>, + linkage: Linkage, +) { + let tcx = cx.tcx; + match mono_item { + MonoItem::Fn(inst) => { + let _inst_guard = + crate::PrintOnPanic(|| format!("{:?} {}", inst, tcx.symbol_name(inst).name)); + debug_assert!(!inst.substs.needs_infer()); + tcx.sess + .time("codegen fn", || crate::base::trans_fn(cx, inst, linkage)); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(&mut cx.constants_cx, def_id); + } + MonoItem::GlobalAsm(hir_id) => { + let item = tcx.hir().expect_item(hir_id); + if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { + cx.global_asm.push_str(&*asm.as_str()); + cx.global_asm.push_str("\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } + } + } +} + +fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { + if std::env::var("CG_CLIF_DISPLAY_CG_TIME") + .as_ref() + .map(|val| &**val) + == Ok("1") + { + println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name); + let before = std::time::Instant::now(); + let res = tcx.sess.time(name, f); + let after = std::time::Instant::now(); + println!( + "[{:<30}: {}] end time: {:?}", + tcx.crate_name(LOCAL_CRATE), + name, + after - before + ); + res + } else { + tcx.sess.time(name, f) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs new file mode 100644 index 00000000000..aa2edb2dfd4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -0,0 +1,293 @@ +//! Codegen of [`asm!`] invocations. + +use crate::prelude::*; + +use std::fmt::Write; + +use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_middle::mir::InlineAsmOperand; +use rustc_target::asm::*; + +pub(crate) fn codegen_inline_asm<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + _span: Span, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperand<'tcx>], + options: InlineAsmOptions, +) { + // FIXME add .eh_frame unwind info directives + + if template.is_empty() { + // Black box + return; + } + + let mut slot_size = Size::from_bytes(0); + let mut clobbered_regs = Vec::new(); + let mut inputs = Vec::new(); + let mut outputs = Vec::new(); + + let mut new_slot = |reg_class: InlineAsmRegClass| { + let reg_size = reg_class + .supported_types(InlineAsmArch::X86_64) + .iter() + .map(|(ty, _)| ty.size()) + .max() + .unwrap(); + let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap(); + slot_size = slot_size.align_to(align); + let offset = slot_size; + slot_size += reg_size; + offset + }; + + // FIXME overlap input and output slots to save stack space + for operand in operands { + match *operand { + InlineAsmOperand::In { reg, ref value } => { + let reg = expect_reg(reg); + clobbered_regs.push((reg, new_slot(reg.reg_class()))); + inputs.push(( + reg, + new_slot(reg.reg_class()), + crate::base::trans_operand(fx, value).load_scalar(fx), + )); + } + InlineAsmOperand::Out { + reg, + late: _, + place, + } => { + let reg = expect_reg(reg); + clobbered_regs.push((reg, new_slot(reg.reg_class()))); + if let Some(place) = place { + outputs.push(( + reg, + new_slot(reg.reg_class()), + crate::base::trans_place(fx, place), + )); + } + } + InlineAsmOperand::InOut { + reg, + late: _, + ref in_value, + out_place, + } => { + let reg = expect_reg(reg); + clobbered_regs.push((reg, new_slot(reg.reg_class()))); + inputs.push(( + reg, + new_slot(reg.reg_class()), + crate::base::trans_operand(fx, in_value).load_scalar(fx), + )); + if let Some(out_place) = out_place { + outputs.push(( + reg, + new_slot(reg.reg_class()), + crate::base::trans_place(fx, out_place), + )); + } + } + InlineAsmOperand::Const { value: _ } => todo!(), + InlineAsmOperand::SymFn { value: _ } => todo!(), + InlineAsmOperand::SymStatic { def_id: _ } => todo!(), + } + } + + let inline_asm_index = fx.inline_asm_index; + fx.inline_asm_index += 1; + let asm_name = format!( + "{}__inline_asm_{}", + fx.tcx.symbol_name(fx.instance).name, + inline_asm_index + ); + + let generated_asm = generate_asm_wrapper( + &asm_name, + InlineAsmArch::X86_64, + options, + template, + clobbered_regs, + &inputs, + &outputs, + ); + fx.cx.global_asm.push_str(&generated_asm); + + call_inline_asm(fx, &asm_name, slot_size, inputs, outputs); +} + +fn generate_asm_wrapper( + asm_name: &str, + arch: InlineAsmArch, + options: InlineAsmOptions, + template: &[InlineAsmTemplatePiece], + clobbered_regs: Vec<(InlineAsmReg, Size)>, + inputs: &[(InlineAsmReg, Size, Value)], + outputs: &[(InlineAsmReg, Size, CPlace<'_>)], +) -> String { + let mut generated_asm = String::new(); + writeln!(generated_asm, ".globl {}", asm_name).unwrap(); + writeln!(generated_asm, ".type {},@function", asm_name).unwrap(); + writeln!( + generated_asm, + ".section .text.{},\"ax\",@progbits", + asm_name + ) + .unwrap(); + writeln!(generated_asm, "{}:", asm_name).unwrap(); + + generated_asm.push_str(".intel_syntax noprefix\n"); + generated_asm.push_str(" push rbp\n"); + generated_asm.push_str(" mov rbp,rdi\n"); + + // Save clobbered registers + if !options.contains(InlineAsmOptions::NORETURN) { + // FIXME skip registers saved by the calling convention + for &(reg, offset) in &clobbered_regs { + save_register(&mut generated_asm, arch, reg, offset); + } + } + + // Write input registers + for &(reg, offset, _value) in inputs { + restore_register(&mut generated_asm, arch, reg, offset); + } + + if options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push_str(".att_syntax\n"); + } + + // The actual inline asm + for piece in template { + match piece { + InlineAsmTemplatePiece::String(s) => { + generated_asm.push_str(s); + } + InlineAsmTemplatePiece::Placeholder { + operand_idx: _, + modifier: _, + span: _, + } => todo!(), + } + } + generated_asm.push('\n'); + + if options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push_str(".intel_syntax noprefix\n"); + } + + if !options.contains(InlineAsmOptions::NORETURN) { + // Read output registers + for &(reg, offset, _place) in outputs { + save_register(&mut generated_asm, arch, reg, offset); + } + + // Restore clobbered registers + for &(reg, offset) in clobbered_regs.iter().rev() { + restore_register(&mut generated_asm, arch, reg, offset); + } + + generated_asm.push_str(" pop rbp\n"); + generated_asm.push_str(" ret\n"); + } else { + generated_asm.push_str(" ud2\n"); + } + + generated_asm.push_str(".att_syntax\n"); + writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap(); + generated_asm.push_str(".text\n"); + generated_asm.push_str("\n\n"); + + generated_asm +} + +fn call_inline_asm<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + asm_name: &str, + slot_size: Size, + inputs: Vec<(InlineAsmReg, Size, Value)>, + outputs: Vec<(InlineAsmReg, Size, CPlace<'tcx>)>, +) { + let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + offset: None, + size: u32::try_from(slot_size.bytes()).unwrap(), + }); + #[cfg(debug_assertions)] + fx.add_comment(stack_slot, "inline asm scratch slot"); + + let inline_asm_func = fx + .cx + .module + .declare_function( + asm_name, + Linkage::Import, + &Signature { + call_conv: CallConv::SystemV, + params: vec![AbiParam::new(fx.pointer_type)], + returns: vec![], + }, + ) + .unwrap(); + let inline_asm_func = fx + .cx + .module + .declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + #[cfg(debug_assertions)] + fx.add_comment(inline_asm_func, asm_name); + + for (_reg, offset, value) in inputs { + fx.bcx + .ins() + .stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap()); + } + + let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0); + fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]); + + for (_reg, offset, place) in outputs { + let ty = fx.clif_type(place.layout().ty).unwrap(); + let value = fx + .bcx + .ins() + .stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap()); + place.write_cvalue(fx, CValue::by_val(value, place.layout())); + } +} + +fn expect_reg(reg_or_class: InlineAsmRegOrRegClass) -> InlineAsmReg { + match reg_or_class { + InlineAsmRegOrRegClass::Reg(reg) => reg, + InlineAsmRegOrRegClass::RegClass(class) => unimplemented!("{:?}", class), + } +} + +fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsmReg, offset: Size) { + match arch { + InlineAsmArch::X86_64 => { + write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap(); + reg.emit(generated_asm, InlineAsmArch::X86_64, None) + .unwrap(); + generated_asm.push('\n'); + } + _ => unimplemented!("save_register for {:?}", arch), + } +} + +fn restore_register( + generated_asm: &mut String, + arch: InlineAsmArch, + reg: InlineAsmReg, + offset: Size, +) { + match arch { + InlineAsmArch::X86_64 => { + generated_asm.push_str(" mov "); + reg.emit(generated_asm, InlineAsmArch::X86_64, None) + .unwrap(); + writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap(); + } + _ => unimplemented!("restore_register for {:?}", arch), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs new file mode 100644 index 00000000000..c1a1cdbe4eb --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs @@ -0,0 +1,93 @@ +//! Emulation of a subset of the cpuid x86 instruction. + +use crate::prelude::*; + +/// Emulates a subset of the cpuid x86 instruction. +/// +/// This emulates an intel cpu with sse and sse2 support, but which doesn't support anything else. +pub(crate) fn codegen_cpuid_call<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + leaf: Value, + _subleaf: Value, +) -> (Value, Value, Value, Value) { + let leaf_0 = fx.bcx.create_block(); + let leaf_1 = fx.bcx.create_block(); + let leaf_8000_0000 = fx.bcx.create_block(); + let leaf_8000_0001 = fx.bcx.create_block(); + let unsupported_leaf = fx.bcx.create_block(); + + let dest = fx.bcx.create_block(); + let eax = fx.bcx.append_block_param(dest, types::I32); + let ebx = fx.bcx.append_block_param(dest, types::I32); + let ecx = fx.bcx.append_block_param(dest, types::I32); + let edx = fx.bcx.append_block_param(dest, types::I32); + + let mut switch = cranelift_frontend::Switch::new(); + switch.set_entry(0, leaf_0); + switch.set_entry(1, leaf_1); + switch.set_entry(0x8000_0000, leaf_8000_0000); + switch.set_entry(0x8000_0001, leaf_8000_0001); + switch.emit(&mut fx.bcx, leaf, unsupported_leaf); + + fx.bcx.switch_to_block(leaf_0); + let max_basic_leaf = fx.bcx.ins().iconst(types::I32, 1); + let vend0 = fx + .bcx + .ins() + .iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu"))); + let vend2 = fx + .bcx + .ins() + .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI"))); + let vend1 = fx + .bcx + .ins() + .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel"))); + fx.bcx + .ins() + .jump(dest, &[max_basic_leaf, vend0, vend1, vend2]); + + fx.bcx.switch_to_block(leaf_1); + let cpu_signature = fx.bcx.ins().iconst(types::I32, 0); + let additional_information = fx.bcx.ins().iconst(types::I32, 0); + let ecx_features = fx.bcx.ins().iconst(types::I32, 0); + let edx_features = fx + .bcx + .ins() + .iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */); + fx.bcx.ins().jump( + dest, + &[ + cpu_signature, + additional_information, + ecx_features, + edx_features, + ], + ); + + fx.bcx.switch_to_block(leaf_8000_0000); + let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0); + let zero = fx.bcx.ins().iconst(types::I32, 0); + fx.bcx + .ins() + .jump(dest, &[extended_max_basic_leaf, zero, zero, zero]); + + fx.bcx.switch_to_block(leaf_8000_0001); + let zero = fx.bcx.ins().iconst(types::I32, 0); + let proc_info_ecx = fx.bcx.ins().iconst(types::I32, 0); + let proc_info_edx = fx.bcx.ins().iconst(types::I32, 0); + fx.bcx + .ins() + .jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]); + + fx.bcx.switch_to_block(unsupported_leaf); + crate::trap::trap_unreachable( + fx, + "__cpuid_count arch intrinsic doesn't yet support specified leaf", + ); + + fx.bcx.switch_to_block(dest); + fx.bcx.ins().nop(); + + (eax, ebx, ecx, edx) +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs new file mode 100644 index 00000000000..18d86f0c5f9 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -0,0 +1,123 @@ +//! Emulate LLVM intrinsics + +use crate::intrinsics::*; +use crate::prelude::*; + +use rustc_middle::ty::subst::SubstsRef; + +pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + intrinsic: &str, + substs: SubstsRef<'tcx>, + args: &[mir::Operand<'tcx>], + destination: Option<(CPlace<'tcx>, BasicBlock)>, +) { + let ret = destination.unwrap().0; + + intrinsic_match! { + fx, intrinsic, substs, args, + _ => { + fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); + crate::trap::trap_unimplemented(fx, intrinsic); + }; + + // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` + llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { + let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout()); + let lane_ty = fx.clif_type(lane_layout.ty).unwrap(); + assert!(lane_count <= 32); + + let mut res = fx.bcx.ins().iconst(types::I32, 0); + + for lane in (0..lane_count).rev() { + let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx); + + // cast float to int + let a_lane = match lane_ty { + types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane), + types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane), + _ => a_lane, + }; + + // extract sign bit of an int + let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1)); + + // shift sign bit into result + let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false); + res = fx.bcx.ins().ishl_imm(res, 1); + res = fx.bcx.ins().bor(res, a_lane_sign); + } + + let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); + ret.write_cvalue(fx, res); + }; + llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) { + let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); + let flt_cc = match kind_const.val.try_to_bits(Size::from_bytes(1)).expect(&format!("kind not scalar: {:?}", kind_const)) { + 0 => FloatCC::Equal, + 1 => FloatCC::LessThan, + 2 => FloatCC::LessThanOrEqual, + 7 => { + unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`."); + } + 3 => { + unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`."); + } + 4 => FloatCC::NotEqual, + 5 => { + unimplemented!("not less than"); + } + 6 => { + unimplemented!("not less than or equal"); + } + kind => unreachable!("kind {:?}", kind), + }; + + simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) + }); + }; + llvm.x86.sse2.psrli.d, (c a, o imm8) { + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); + simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { + let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).expect(&format!("imm8 not scalar: {:?}", imm8)) { + imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }; + CValue::by_val(res_lane, res_lane_layout) + }); + }; + llvm.x86.sse2.pslli.d, (c a, o imm8) { + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); + simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { + let res_lane = match imm8.val.try_to_bits(Size::from_bytes(4)).expect(&format!("imm8 not scalar: {:?}", imm8)) { + imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), + }; + CValue::by_val(res_lane, res_lane_layout) + }); + }; + llvm.x86.sse2.storeu.dq, (v mem_addr, c a) { + // FIXME correctly handle the unalignment + let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); + dest.write_cvalue(fx, a); + }; + } + + if let Some((_, dest)) = destination { + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + } else { + trap_unreachable(fx, "[corruption] Diverging intrinsic returned."); + } +} + +// llvm.x86.avx2.vperm2i128 +// llvm.x86.ssse3.pshuf.b.128 +// llvm.x86.avx2.pshuf.b +// llvm.x86.avx2.psrli.w +// llvm.x86.sse2.psrli.w diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs new file mode 100644 index 00000000000..9a3e4c7b56e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -0,0 +1,1099 @@ +//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"` +//! and LLVM intrinsics that have symbol names starting with `llvm.`. + +mod cpuid; +mod llvm; +mod simd; + +pub(crate) use cpuid::codegen_cpuid_call; +pub(crate) use llvm::codegen_llvm_intrinsic_call; + +use crate::prelude::*; + +macro intrinsic_pat { + (_) => { + _ + }, + ($name:ident) => { + stringify!($name) + }, + ($name:literal) => { + stringify!($name) + }, + ($x:ident . $($xs:tt).*) => { + concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) + } +} + +macro intrinsic_arg { + (o $fx:expr, $arg:ident) => { + $arg + }, + (c $fx:expr, $arg:ident) => { + trans_operand($fx, $arg) + }, + (v $fx:expr, $arg:ident) => { + trans_operand($fx, $arg).load_scalar($fx) + } +} + +macro intrinsic_substs { + ($substs:expr, $index:expr,) => {}, + ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => { + let $first = $substs.type_at($index); + intrinsic_substs!($substs, $index+1, $($rest),*); + } +} + +macro intrinsic_match { + ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr, + _ => $unknown:block; + $( + $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block; + )*) => { + let _ = $substs; // Silence warning when substs is unused. + match $intrinsic { + $( + $(intrinsic_pat!($($name).*))|* $(if $cond)? => { + #[allow(unused_parens, non_snake_case)] + { + $( + intrinsic_substs!($substs, 0, $($subst),*); + )? + if let [$($arg),*] = $args { + let ($($arg,)*) = ( + $(intrinsic_arg!($a $fx, $arg),)* + ); + #[warn(unused_parens, non_snake_case)] + { + $content + } + } else { + bug!("wrong number of args for intrinsic {:?}", $intrinsic); + } + } + } + )* + _ => $unknown, + } + } +} + +macro call_intrinsic_match { + ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $( + $name:ident($($arg:ident),*) -> $ty:ident => $func:ident, + )*) => { + match $intrinsic { + $( + stringify!($name) => { + assert!($substs.is_noop()); + if let [$(ref $arg),*] = *$args { + let ($($arg,)*) = ( + $(trans_operand($fx, $arg),)* + ); + let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty); + $ret.write_cvalue($fx, res); + + if let Some((_, dest)) = $destination { + let ret_block = $fx.get_block(dest); + $fx.bcx.ins().jump(ret_block, &[]); + return; + } else { + unreachable!(); + } + } else { + bug!("wrong number of args for intrinsic {:?}", $intrinsic); + } + } + )* + _ => {} + } + } +} + +macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) { + crate::atomic_shim::lock_global_lock($fx); + + let clif_ty = $fx.clif_type($T).unwrap(); + let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0); + let new = $fx.bcx.ins().$op(old, $src); + $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0); + $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T))); + + crate::atomic_shim::unlock_global_lock($fx); +} + +macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) { + crate::atomic_shim::lock_global_lock($fx); + + // Read old + let clif_ty = $fx.clif_type($T).unwrap(); + let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0); + + // Compare + let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src); + let new = $fx.bcx.ins().select(is_eq, old, $src); + + // Write new + $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0); + + let ret_val = CValue::by_val(old, $ret.layout()); + $ret.write_cvalue($fx, ret_val); + + crate::atomic_shim::unlock_global_lock($fx); +} + +macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { + match $ty.kind() { + ty::Uint(_) | ty::Int(_) => {} + _ => { + $fx.tcx.sess.span_err( + $span, + &format!( + "`{}` intrinsic: expected basic integer type, found `{:?}`", + $intrinsic, $ty + ), + ); + // Prevent verifier error + crate::trap::trap_unreachable($fx, "compilation should not have succeeded"); + return; + } + } +} + +macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) { + if !$ty.is_simd() { + $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty)); + // Prevent verifier error + crate::trap::trap_unreachable($fx, "compilation should not have succeeded"); + return; + } +} + +fn lane_type_and_count<'tcx>( + tcx: TyCtxt<'tcx>, + layout: TyAndLayout<'tcx>, +) -> (TyAndLayout<'tcx>, u16) { + assert!(layout.ty.is_simd()); + let lane_count = match layout.fields { + rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(), + _ => unreachable!("lane_type_and_count({:?})", layout), + }; + let lane_layout = layout + .field( + &ty::layout::LayoutCx { + tcx, + param_env: ParamEnv::reveal_all(), + }, + 0, + ) + .unwrap(); + (lane_layout, lane_count) +} + +pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> { + let (element, count) = match &layout.abi { + Abi::Vector { element, count } => (element.clone(), *count), + _ => unreachable!(), + }; + + match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) { + // Cranelift currently only implements icmp for 128bit vectors. + Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty), + _ => None, + } +} + +fn simd_for_each_lane<'tcx, M: Module>( + fx: &mut FunctionCx<'_, 'tcx, M>, + val: CValue<'tcx>, + ret: CPlace<'tcx>, + f: impl Fn( + &mut FunctionCx<'_, 'tcx, M>, + TyAndLayout<'tcx>, + TyAndLayout<'tcx>, + Value, + ) -> CValue<'tcx>, +) { + let layout = val.layout(); + + let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); + let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + assert_eq!(lane_count, ret_lane_count); + + for lane_idx in 0..lane_count { + let lane_idx = mir::Field::new(lane_idx.try_into().unwrap()); + let lane = val.value_field(fx, lane_idx).load_scalar(fx); + + let res_lane = f(fx, lane_layout, ret_lane_layout, lane); + + ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane); + } +} + +fn simd_pair_for_each_lane<'tcx, M: Module>( + fx: &mut FunctionCx<'_, 'tcx, M>, + x: CValue<'tcx>, + y: CValue<'tcx>, + ret: CPlace<'tcx>, + f: impl Fn( + &mut FunctionCx<'_, 'tcx, M>, + TyAndLayout<'tcx>, + TyAndLayout<'tcx>, + Value, + Value, + ) -> CValue<'tcx>, +) { + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); + let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + assert_eq!(lane_count, ret_lane_count); + + for lane in 0..lane_count { + let lane = mir::Field::new(lane.try_into().unwrap()); + let x_lane = x.value_field(fx, lane).load_scalar(fx); + let y_lane = y.value_field(fx, lane).load_scalar(fx); + + let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane); + + ret.place_field(fx, lane).write_cvalue(fx, res_lane); + } +} + +fn bool_to_zero_or_max_uint<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + val: Value, +) -> CValue<'tcx> { + let ty = fx.clif_type(layout.ty).unwrap(); + + let int_ty = match ty { + types::F32 => types::I32, + types::F64 => types::I64, + ty => ty, + }; + + let val = fx.bcx.ins().bint(int_ty, val); + let mut res = fx.bcx.ins().ineg(val); + + if ty.is_float() { + res = fx.bcx.ins().bitcast(ty, res); + } + + CValue::by_val(res, layout) +} + +macro simd_cmp { + ($fx:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => { + let vector_ty = clif_vector_type($fx.tcx, $x.layout()); + + if let Some(vector_ty) = vector_ty { + let x = $x.load_scalar($fx); + let y = $y.load_scalar($fx); + let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y); + + // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1. + let val = $fx.bcx.ins().raw_bitcast(vector_ty, val); + + $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout())); + } else { + simd_pair_for_each_lane( + $fx, + $x, + $y, + $ret, + |fx, lane_layout, res_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) + }, + ); + } + }, + ($fx:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => { + // FIXME use vector icmp when possible + simd_pair_for_each_lane( + $fx, + $x, + $y, + $ret, + |fx, lane_layout, res_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane), + ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) + }, + ); + }, +} + +macro simd_int_binop { + ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => { + simd_int_binop!($fx, $op|$op($x, $y) -> $ret); + }, + ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => { + simd_pair_for_each_lane( + $fx, + $x, + $y, + $ret, + |fx, lane_layout, ret_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane), + ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + CValue::by_val(res_lane, ret_lane_layout) + }, + ); + }, +} + +macro simd_int_flt_binop { + ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => { + simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret); + }, + ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => { + simd_pair_for_each_lane( + $fx, + $x, + $y, + $ret, + |fx, lane_layout, ret_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane), + ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane), + ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + CValue::by_val(res_lane, ret_lane_layout) + }, + ); + }, +} + +macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) { + simd_pair_for_each_lane( + $fx, + $x, + $y, + $ret, + |fx, lane_layout, ret_lane_layout, x_lane, y_lane| { + let res_lane = match lane_layout.ty.kind() { + ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane), + _ => unreachable!("{:?}", lane_layout.ty), + }; + CValue::by_val(res_lane, ret_lane_layout) + }, + ); +} + +pub(crate) fn codegen_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + instance: Instance<'tcx>, + args: &[mir::Operand<'tcx>], + destination: Option<(CPlace<'tcx>, BasicBlock)>, + span: Span, +) { + let def_id = instance.def_id(); + let substs = instance.substs; + + let intrinsic = fx.tcx.item_name(def_id).as_str(); + let intrinsic = &intrinsic[..]; + + let ret = match destination { + Some((place, _)) => place, + None => { + // Insert non returning intrinsics here + match intrinsic { + "abort" => { + trap_abort(fx, "Called intrinsic::abort."); + } + "unreachable" => { + trap_unreachable(fx, "[corruption] Called intrinsic::unreachable."); + } + "transmute" => { + crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); + } + _ => unimplemented!("unsupported instrinsic {}", intrinsic), + } + return; + } + }; + + if intrinsic.starts_with("simd_") { + self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); + let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); + fx.bcx.ins().jump(ret_block, &[]); + return; + } + + let usize_layout = fx.layout_of(fx.tcx.types.usize); + + call_intrinsic_match! { + fx, intrinsic, substs, ret, destination, args, + expf32(flt) -> f32 => expf, + expf64(flt) -> f64 => exp, + exp2f32(flt) -> f32 => exp2f, + exp2f64(flt) -> f64 => exp2, + sqrtf32(flt) -> f32 => sqrtf, + sqrtf64(flt) -> f64 => sqrt, + powif32(a, x) -> f32 => __powisf2, // compiler-builtins + powif64(a, x) -> f64 => __powidf2, // compiler-builtins + powf32(a, x) -> f32 => powf, + powf64(a, x) -> f64 => pow, + logf32(flt) -> f32 => logf, + logf64(flt) -> f64 => log, + log2f32(flt) -> f32 => log2f, + log2f64(flt) -> f64 => log2, + log10f32(flt) -> f32 => log10f, + log10f64(flt) -> f64 => log10, + fabsf32(flt) -> f32 => fabsf, + fabsf64(flt) -> f64 => fabs, + fmaf32(x, y, z) -> f32 => fmaf, + fmaf64(x, y, z) -> f64 => fma, + copysignf32(x, y) -> f32 => copysignf, + copysignf64(x, y) -> f64 => copysign, + + // rounding variants + // FIXME use clif insts + floorf32(flt) -> f32 => floorf, + floorf64(flt) -> f64 => floor, + ceilf32(flt) -> f32 => ceilf, + ceilf64(flt) -> f64 => ceil, + truncf32(flt) -> f32 => truncf, + truncf64(flt) -> f64 => trunc, + roundf32(flt) -> f32 => roundf, + roundf64(flt) -> f64 => round, + + // trigonometry + sinf32(flt) -> f32 => sinf, + sinf64(flt) -> f64 => sin, + cosf32(flt) -> f32 => cosf, + cosf64(flt) -> f64 => cos, + tanf32(flt) -> f32 => tanf, + tanf64(flt) -> f64 => tan, + } + + intrinsic_match! { + fx, intrinsic, substs, args, + _ => { + fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic)); + }; + + assume, (c _a) {}; + likely | unlikely, (c a) { + ret.write_cvalue(fx, a); + }; + breakpoint, () { + fx.bcx.ins().debugtrap(); + }; + copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) { + let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); + let elem_size = fx + .bcx + .ins() + .iconst(fx.pointer_type, elem_size as i64); + assert_eq!(args.len(), 3); + let byte_amount = fx.bcx.ins().imul(count, elem_size); + + if intrinsic.contains("nonoverlapping") { + // FIXME emit_small_memcpy + fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + } else { + // FIXME emit_small_memmove + fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + } + }; + // NOTE: the volatile variants have src and dst swapped + volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) { + let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); + let elem_size = fx + .bcx + .ins() + .iconst(fx.pointer_type, elem_size as i64); + assert_eq!(args.len(), 3); + let byte_amount = fx.bcx.ins().imul(count, elem_size); + + // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} + if intrinsic.contains("nonoverlapping") { + // FIXME emit_small_memcpy + fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + } else { + // FIXME emit_small_memmove + fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + } + }; + discriminant_value, (c ptr) { + let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); + let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); + ret.write_cvalue(fx, discr); + }; + size_of_val, <T> (c ptr) { + let layout = fx.layout_of(T); + let size = if layout.is_unsized() { + let (_ptr, info) = ptr.load_scalar_pair(fx); + let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info); + size + } else { + fx + .bcx + .ins() + .iconst(fx.pointer_type, layout.size.bytes() as i64) + }; + ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); + }; + min_align_of_val, <T> (c ptr) { + let layout = fx.layout_of(T); + let align = if layout.is_unsized() { + let (_ptr, info) = ptr.load_scalar_pair(fx); + let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info); + align + } else { + fx + .bcx + .ins() + .iconst(fx.pointer_type, layout.align.abi.bytes() as i64) + }; + ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); + }; + + _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + // FIXME trap on overflow + let bin_op = match intrinsic { + "unchecked_add" => BinOp::Add, + "unchecked_sub" => BinOp::Sub, + "unchecked_div" | "exact_div" => BinOp::Div, + "unchecked_rem" => BinOp::Rem, + "unchecked_shl" => BinOp::Shl, + "unchecked_shr" => BinOp::Shr, + _ => unreachable!("intrinsic {}", intrinsic), + }; + let res = crate::num::trans_int_binop(fx, bin_op, x, y); + ret.write_cvalue(fx, res); + }; + _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + assert_eq!(x.layout().ty, y.layout().ty); + let bin_op = match intrinsic { + "add_with_overflow" => BinOp::Add, + "sub_with_overflow" => BinOp::Sub, + "mul_with_overflow" => BinOp::Mul, + _ => unreachable!("intrinsic {}", intrinsic), + }; + + let res = crate::num::trans_checked_int_binop( + fx, + bin_op, + x, + y, + ); + ret.write_cvalue(fx, res); + }; + _ if intrinsic.starts_with("wrapping_"), (c x, c y) { + assert_eq!(x.layout().ty, y.layout().ty); + let bin_op = match intrinsic { + "wrapping_add" => BinOp::Add, + "wrapping_sub" => BinOp::Sub, + "wrapping_mul" => BinOp::Mul, + _ => unreachable!("intrinsic {}", intrinsic), + }; + let res = crate::num::trans_int_binop( + fx, + bin_op, + x, + y, + ); + ret.write_cvalue(fx, res); + }; + _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) { + assert_eq!(lhs.layout().ty, rhs.layout().ty); + let bin_op = match intrinsic { + "saturating_add" => BinOp::Add, + "saturating_sub" => BinOp::Sub, + _ => unreachable!("intrinsic {}", intrinsic), + }; + + let signed = type_sign(T); + + let checked_res = crate::num::trans_checked_int_binop( + fx, + bin_op, + lhs, + rhs, + ); + + let (val, has_overflow) = checked_res.load_scalar_pair(fx); + let clif_ty = fx.clif_type(T).unwrap(); + + // `select.i8` is not implemented by Cranelift. + let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow); + + let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); + + let val = match (intrinsic, signed) { + ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), + ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), + ("saturating_add", true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + ("saturating_sub", true) => { + let rhs = rhs.load_scalar(fx); + let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); + fx.bcx.ins().select(has_overflow, sat_val, val) + } + _ => unreachable!(), + }; + + let res = CValue::by_val(val, fx.layout_of(T)); + + ret.write_cvalue(fx, res); + }; + rotate_left, <T>(v x, v y) { + let layout = fx.layout_of(T); + let res = fx.bcx.ins().rotl(x, y); + ret.write_cvalue(fx, CValue::by_val(res, layout)); + }; + rotate_right, <T>(v x, v y) { + let layout = fx.layout_of(T); + let res = fx.bcx.ins().rotr(x, y); + ret.write_cvalue(fx, CValue::by_val(res, layout)); + }; + + // The only difference between offset and arith_offset is regarding UB. Because Cranelift + // doesn't have UB both are codegen'ed the same way + offset | arith_offset, (c base, v offset) { + let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty; + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); + let base_val = base.load_scalar(fx); + let res = fx.bcx.ins().iadd(base_val, ptr_diff); + ret.write_cvalue(fx, CValue::by_val(res, base.layout())); + }; + + transmute, (c from) { + ret.write_cvalue_transmute(fx, from); + }; + write_bytes | volatile_set_memory, (c dst, v val, v count) { + let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty; + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let count = fx.bcx.ins().imul_imm(count, pointee_size as i64); + let dst_ptr = dst.load_scalar(fx); + // FIXME make the memset actually volatile when switching to emit_small_memset + // FIXME use emit_small_memset + fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + }; + ctlz | ctlz_nonzero, <T> (v arg) { + // FIXME trap on `ctlz_nonzero` with zero arg. + let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 { + // FIXME verify this algorithm is correct + let (lsb, msb) = fx.bcx.ins().isplit(arg); + let lsb_lz = fx.bcx.ins().clz(lsb); + let msb_lz = fx.bcx.ins().clz(msb); + let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0); + let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64); + let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz); + fx.bcx.ins().uextend(types::I128, res) + } else { + fx.bcx.ins().clz(arg) + }; + let res = CValue::by_val(res, fx.layout_of(T)); + ret.write_cvalue(fx, res); + }; + cttz | cttz_nonzero, <T> (v arg) { + // FIXME trap on `cttz_nonzero` with zero arg. + let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 { + // FIXME verify this algorithm is correct + let (lsb, msb) = fx.bcx.ins().isplit(arg); + let lsb_tz = fx.bcx.ins().ctz(lsb); + let msb_tz = fx.bcx.ins().ctz(msb); + let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0); + let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64); + let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz); + fx.bcx.ins().uextend(types::I128, res) + } else { + fx.bcx.ins().ctz(arg) + }; + let res = CValue::by_val(res, fx.layout_of(T)); + ret.write_cvalue(fx, res); + }; + ctpop, <T> (v arg) { + let res = fx.bcx.ins().popcnt(arg); + let res = CValue::by_val(res, fx.layout_of(T)); + ret.write_cvalue(fx, res); + }; + bitreverse, <T> (v arg) { + let res = fx.bcx.ins().bitrev(arg); + let res = CValue::by_val(res, fx.layout_of(T)); + ret.write_cvalue(fx, res); + }; + bswap, <T> (v arg) { + // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift + fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { + match bcx.func.dfg.value_type(v) { + types::I8 => v, + + // https://code.woboq.org/gcc/include/bits/byteswap.h.html + types::I16 => { + let tmp1 = bcx.ins().ishl_imm(v, 8); + let n1 = bcx.ins().band_imm(tmp1, 0xFF00); + + let tmp2 = bcx.ins().ushr_imm(v, 8); + let n2 = bcx.ins().band_imm(tmp2, 0x00FF); + + bcx.ins().bor(n1, n2) + } + types::I32 => { + let tmp1 = bcx.ins().ishl_imm(v, 24); + let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000); + + let tmp2 = bcx.ins().ishl_imm(v, 8); + let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000); + + let tmp3 = bcx.ins().ushr_imm(v, 8); + let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00); + + let tmp4 = bcx.ins().ushr_imm(v, 24); + let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF); + + let or_tmp1 = bcx.ins().bor(n1, n2); + let or_tmp2 = bcx.ins().bor(n3, n4); + bcx.ins().bor(or_tmp1, or_tmp2) + } + types::I64 => { + let tmp1 = bcx.ins().ishl_imm(v, 56); + let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64); + + let tmp2 = bcx.ins().ishl_imm(v, 40); + let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64); + + let tmp3 = bcx.ins().ishl_imm(v, 24); + let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64); + + let tmp4 = bcx.ins().ishl_imm(v, 8); + let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64); + + let tmp5 = bcx.ins().ushr_imm(v, 8); + let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64); + + let tmp6 = bcx.ins().ushr_imm(v, 24); + let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64); + + let tmp7 = bcx.ins().ushr_imm(v, 40); + let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64); + + let tmp8 = bcx.ins().ushr_imm(v, 56); + let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64); + + let or_tmp1 = bcx.ins().bor(n1, n2); + let or_tmp2 = bcx.ins().bor(n3, n4); + let or_tmp3 = bcx.ins().bor(n5, n6); + let or_tmp4 = bcx.ins().bor(n7, n8); + + let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2); + let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4); + bcx.ins().bor(or_tmp5, or_tmp6) + } + types::I128 => { + let (lo, hi) = bcx.ins().isplit(v); + let lo = swap(bcx, lo); + let hi = swap(bcx, hi); + bcx.ins().iconcat(hi, lo) + } + ty => unreachable!("bswap {}", ty), + } + }; + let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T)); + ret.write_cvalue(fx, res); + }; + assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () { + let layout = fx.layout_of(T); + if layout.abi.is_uninhabited() { + crate::base::codegen_panic( + fx, + &format!("attempted to instantiate uninhabited type `{}`", T), + span, + ); + return; + } + + if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + crate::base::codegen_panic( + fx, + &format!("attempted to zero-initialize type `{}`, which is invalid", T), + span, + ); + return; + } + + if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + crate::base::codegen_panic( + fx, + &format!("attempted to leave type `{}` uninitialized, which is invalid", T), + span, + ); + return; + } + }; + + volatile_load | unaligned_volatile_load, (c ptr) { + // Cranelift treats loads as volatile by default + // FIXME ignore during stack2reg optimization + // FIXME correctly handle unaligned_volatile_load + let inner_layout = + fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); + ret.write_cvalue(fx, val); + }; + volatile_store | unaligned_volatile_store, (v ptr, c val) { + // Cranelift treats stores as volatile by default + // FIXME ignore during stack2reg optimization + // FIXME correctly handle unaligned_volatile_store + let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); + dest.write_cvalue(fx, val); + }; + + size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () { + let const_val = + fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); + let val = crate::constant::trans_const_value( + fx, + const_val, + ret.layout().ty, + ); + ret.write_cvalue(fx, val); + }; + + ptr_offset_from, <T> (v ptr, v base) { + let isize_layout = fx.layout_of(fx.tcx.types.isize); + + let pointee_size: u64 = fx.layout_of(T).size.bytes(); + let diff = fx.bcx.ins().isub(ptr, base); + // FIXME this can be an exact division. + let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout); + ret.write_cvalue(fx, val); + }; + + ptr_guaranteed_eq, (c a, c b) { + let val = crate::num::trans_ptr_binop(fx, BinOp::Eq, a, b); + ret.write_cvalue(fx, val); + }; + + ptr_guaranteed_ne, (c a, c b) { + let val = crate::num::trans_ptr_binop(fx, BinOp::Ne, a, b); + ret.write_cvalue(fx, val); + }; + + caller_location, () { + let caller_location = fx.get_caller_location(span); + ret.write_cvalue(fx, caller_location); + }; + + _ if intrinsic.starts_with("atomic_fence"), () { + crate::atomic_shim::lock_global_lock(fx); + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_singlethreadfence"), () { + crate::atomic_shim::lock_global_lock(fx); + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_load"), (c ptr) { + crate::atomic_shim::lock_global_lock(fx); + + let inner_layout = + fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); + validate_atomic_type!(fx, intrinsic, span, inner_layout.ty); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); + ret.write_cvalue(fx, val); + + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + validate_atomic_type!(fx, intrinsic, span, val.layout().ty); + + crate::atomic_shim::lock_global_lock(fx); + + let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); + dest.write_cvalue(fx, val); + + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, T); + + crate::atomic_shim::lock_global_lock(fx); + + // Read old + let clif_ty = fx.clif_type(T).unwrap(); + let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); + ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T))); + + // Write new + let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout()); + dest.write_cvalue(fx, src); + + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + validate_atomic_type!(fx, intrinsic, span, T); + + let test_old = test_old.load_scalar(fx); + let new = new.load_scalar(fx); + + crate::atomic_shim::lock_global_lock(fx); + + // Read old + let clif_ty = fx.clif_type(T).unwrap(); + let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); + + // Compare + let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old); + let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old + + // Write new + fx.bcx.ins().store(MemFlags::new(), new, ptr, 0); + + let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout()); + ret.write_cvalue(fx, ret_val); + + crate::atomic_shim::unlock_global_lock(fx); + }; + + _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let amount = amount.load_scalar(fx); + atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret); + }; + _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let amount = amount.load_scalar(fx); + atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret); + }; + _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret); + }; + _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, T); + + let src = src.load_scalar(fx); + + crate::atomic_shim::lock_global_lock(fx); + + let clif_ty = fx.clif_type(T).unwrap(); + let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0); + let and = fx.bcx.ins().band(old, src); + let new = fx.bcx.ins().bnot(and); + fx.bcx.ins().store(MemFlags::new(), new, ptr, 0); + ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T))); + + crate::atomic_shim::unlock_global_lock(fx); + }; + _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret); + }; + _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret); + }; + + _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret); + }; + _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret); + }; + _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret); + }; + _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) { + validate_atomic_type!(fx, intrinsic, span, ret.layout().ty); + let src = src.load_scalar(fx); + atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret); + }; + + minnumf32, (v a, v b) { + let val = fx.bcx.ins().fmin(a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); + ret.write_cvalue(fx, val); + }; + minnumf64, (v a, v b) { + let val = fx.bcx.ins().fmin(a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); + ret.write_cvalue(fx, val); + }; + maxnumf32, (v a, v b) { + let val = fx.bcx.ins().fmax(a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); + ret.write_cvalue(fx, val); + }; + maxnumf64, (v a, v b) { + let val = fx.bcx.ins().fmax(a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); + ret.write_cvalue(fx, val); + }; + + try, (v f, v data, v _catch_fn) { + // FIXME once unwinding is supported, change this to actually catch panics + let f_sig = fx.bcx.func.import_signature(Signature { + call_conv: CallConv::triple_default(fx.triple()), + params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))], + returns: vec![], + }); + + fx.bcx.ins().call_indirect(f_sig, f, &[data]); + + let ret_val = CValue::const_val(fx, ret.layout(), 0); + ret.write_cvalue(fx, ret_val); + }; + + fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { + let res = crate::num::trans_float_binop(fx, match intrinsic { + "fadd_fast" => BinOp::Add, + "fsub_fast" => BinOp::Sub, + "fmul_fast" => BinOp::Mul, + "fdiv_fast" => BinOp::Div, + "frem_fast" => BinOp::Rem, + _ => unreachable!(), + }, x, y); + ret.write_cvalue(fx, res); + }; + float_to_int_unchecked, (v f) { + let res = crate::cast::clif_int_or_float_cast( + fx, + f, + false, + fx.clif_type(ret.layout().ty).unwrap(), + type_sign(ret.layout().ty), + ); + ret.write_cvalue(fx, CValue::by_val(res, ret.layout())); + }; + } + + if let Some((_, dest)) = destination { + let ret_block = fx.get_block(dest); + fx.bcx.ins().jump(ret_block, &[]); + } else { + trap_unreachable(fx, "[corruption] Diverging intrinsic returned."); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs new file mode 100644 index 00000000000..b4269f4fafa --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -0,0 +1,238 @@ +//! Codegen `extern "platform-intrinsic"` intrinsics. + +use super::*; +use crate::prelude::*; + +pub(super) fn codegen_simd_intrinsic_call<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + instance: Instance<'tcx>, + args: &[mir::Operand<'tcx>], + ret: CPlace<'tcx>, + span: Span, +) { + let def_id = instance.def_id(); + let substs = instance.substs; + + let intrinsic = fx.tcx.item_name(def_id).as_str(); + let intrinsic = &intrinsic[..]; + + intrinsic_match! { + fx, intrinsic, substs, args, + _ => { + fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); + }; + + simd_cast, (c a) { + validate_simd_type!(fx, intrinsic, span, a.layout().ty); + simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| { + let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap(); + + let from_signed = type_sign(lane_layout.ty); + let to_signed = type_sign(ret_lane_layout.ty); + + let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed); + CValue::by_val(ret_lane, ret_lane_layout) + }); + }; + + // FIXME support float comparisons + simd_eq, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, Equal(x, y) -> ret); + }; + simd_ne, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, NotEqual(x, y) -> ret); + }; + simd_lt, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, UnsignedLessThan|SignedLessThan(x, y) -> ret); + }; + simd_le, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual(x, y) -> ret); + }; + simd_gt, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan(x, y) -> ret); + }; + simd_ge, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_cmp!(fx, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual(x, y) -> ret); + }; + + // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U + _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + + let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); + + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout); + let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + + assert_eq!(lane_type, ret_lane_type); + assert_eq!(n, ret_lane_count); + + let total_len = lane_count * 2; + + let indexes = { + use rustc_middle::mir::interpret::*; + let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const"); + + let idx_bytes = match idx_const.val { + ty::ConstKind::Value(ConstValue::ByRef { alloc, offset }) => { + let ptr = Pointer::new(AllocId(0 /* dummy */), offset); + let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */); + alloc.get_bytes(fx, ptr, size).unwrap() + } + _ => unreachable!("{:?}", idx_const), + }; + + (0..ret_lane_count).map(|i| { + let i = usize::try_from(i).unwrap(); + let idx = rustc_middle::mir::interpret::read_target_uint( + fx.tcx.data_layout.endian, + &idx_bytes[4*i.. 4*i + 4], + ).expect("read_target_uint"); + u16::try_from(idx).expect("try_from u32") + }).collect::<Vec<u16>>() + }; + + for &idx in &indexes { + assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len); + } + + for (out_idx, in_idx) in indexes.into_iter().enumerate() { + let in_lane = if in_idx < lane_count { + x.value_field(fx, mir::Field::new(in_idx.try_into().unwrap())) + } else { + y.value_field(fx, mir::Field::new((in_idx - lane_count).try_into().unwrap())) + }; + let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); + out_lane.write_cvalue(fx, in_lane); + } + }; + + simd_insert, (c base, o idx, c val) { + // FIXME validate + let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { + idx_const + } else { + fx.tcx.sess.span_fatal( + span, + "Index argument for `simd_insert` is not a constant", + ); + }; + + let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).expect(&format!("kind not scalar: {:?}", idx_const)); + let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout()); + if idx >= lane_count.into() { + fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); + } + + ret.write_cvalue(fx, base); + let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap())); + ret_lane.write_cvalue(fx, val); + }; + + simd_extract, (c v, o idx) { + validate_simd_type!(fx, intrinsic, span, v.layout().ty); + let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { + idx_const + } else { + fx.tcx.sess.span_fatal( + span, + "Index argument for `simd_extract` is not a constant", + ); + }; + + let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).expect(&format!("kind not scalar: {:?}", idx_const)); + let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout()); + if idx >= lane_count.into() { + fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); + } + + let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap())); + ret.write_cvalue(fx, ret_lane); + }; + + simd_add, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret); + }; + simd_sub, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret); + }; + simd_mul, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret); + }; + simd_div, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret); + }; + simd_shl, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_binop!(fx, ishl(x, y) -> ret); + }; + simd_shr, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_binop!(fx, ushr|sshr(x, y) -> ret); + }; + simd_and, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_binop!(fx, band(x, y) -> ret); + }; + simd_or, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_binop!(fx, bor(x, y) -> ret); + }; + simd_xor, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_int_binop!(fx, bxor(x, y) -> ret); + }; + + simd_fma, (c a, c b, c c) { + validate_simd_type!(fx, intrinsic, span, a.layout().ty); + assert_eq!(a.layout(), b.layout()); + assert_eq!(a.layout(), c.layout()); + let layout = a.layout(); + + let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout); + let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout()); + assert_eq!(lane_count, ret_lane_count); + + for lane in 0..lane_count { + let lane = mir::Field::new(lane.try_into().unwrap()); + let a_lane = a.value_field(fx, lane).load_scalar(fx); + let b_lane = b.value_field(fx, lane).load_scalar(fx); + let c_lane = c.value_field(fx, lane).load_scalar(fx); + + let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane); + let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout); + + ret.place_field(fx, lane).write_cvalue(fx, res_lane); + } + }; + + simd_fmin, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_flt_binop!(fx, fmin(x, y) -> ret); + }; + simd_fmax, (c x, c y) { + validate_simd_type!(fx, intrinsic, span, x.layout().ty); + simd_flt_binop!(fx, fmax(x, y) -> ret); + }; + + // simd_fabs + // simd_saturating_add + // simd_bitmask + // simd_select + // simd_reduce_add_{,un}ordered + // simd_rem + } +} diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs new file mode 100644 index 00000000000..fd00a2e00a6 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -0,0 +1,316 @@ +#![feature( + rustc_private, + decl_macro, + type_alias_impl_trait, + associated_type_bounds, + never_type, + try_blocks, + hash_drain_filter +)] +#![warn(rust_2018_idioms)] +#![warn(unused_lifetimes)] + +#[cfg(feature = "jit")] +extern crate libc; +extern crate snap; +#[macro_use] +extern crate rustc_middle; +extern crate rustc_ast; +extern crate rustc_codegen_ssa; +extern crate rustc_data_structures; +extern crate rustc_errors; +extern crate rustc_fs_util; +extern crate rustc_hir; +extern crate rustc_incremental; +extern crate rustc_index; +extern crate rustc_session; +extern crate rustc_span; +extern crate rustc_symbol_mangling; +extern crate rustc_target; + +// This prevents duplicating functions and statics that are already part of the host rustc process. +#[allow(unused_extern_crates)] +extern crate rustc_driver; + +use std::any::Any; + +use rustc_codegen_ssa::traits::CodegenBackend; +use rustc_codegen_ssa::CodegenResults; +use rustc_errors::ErrorReported; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; +use rustc_middle::ty::query::Providers; +use rustc_session::config::OutputFilenames; +use rustc_session::Session; + +use cranelift_codegen::settings::{self, Configurable}; + +use crate::constant::ConstantCx; +use crate::prelude::*; + +mod abi; +mod allocator; +mod analyze; +mod archive; +mod atomic_shim; +mod backend; +mod base; +mod cast; +mod codegen_i128; +mod common; +mod constant; +mod debuginfo; +mod discriminant; +mod driver; +mod inline_asm; +mod intrinsics; +mod linkage; +mod main_shim; +mod metadata; +mod num; +mod optimize; +mod pointer; +mod pretty_clif; +mod toolchain; +mod trap; +mod unsize; +mod value_and_place; +mod vtable; + +mod prelude { + pub(crate) use std::convert::{TryFrom, TryInto}; + + pub(crate) use rustc_ast::ast::{FloatTy, IntTy, UintTy}; + pub(crate) use rustc_span::Span; + + pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE}; + pub(crate) use rustc_middle::bug; + pub(crate) use rustc_middle::mir::{self, *}; + pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout}; + pub(crate) use rustc_middle::ty::{ + self, FnSig, Instance, InstanceDef, ParamEnv, Ty, TyCtxt, TypeAndMut, TypeFoldable, + }; + pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx}; + + pub(crate) use rustc_data_structures::fx::FxHashMap; + + pub(crate) use rustc_index::vec::Idx; + + pub(crate) use cranelift_codegen::entity::EntitySet; + pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; + pub(crate) use cranelift_codegen::ir::function::Function; + pub(crate) use cranelift_codegen::ir::types; + pub(crate) use cranelift_codegen::ir::{ + AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, + StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value, + }; + pub(crate) use cranelift_codegen::isa::{self, CallConv}; + pub(crate) use cranelift_codegen::Context; + pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; + pub(crate) use cranelift_module::{self, DataContext, DataId, FuncId, Linkage, Module}; + + pub(crate) use crate::abi::*; + pub(crate) use crate::base::{trans_operand, trans_place}; + pub(crate) use crate::cast::*; + pub(crate) use crate::common::*; + pub(crate) use crate::debuginfo::{DebugContext, UnwindContext}; + pub(crate) use crate::pointer::Pointer; + pub(crate) use crate::trap::*; + pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue}; +} + +struct PrintOnPanic<F: Fn() -> String>(F); +impl<F: Fn() -> String> Drop for PrintOnPanic<F> { + fn drop(&mut self) { + if ::std::thread::panicking() { + println!("{}", (self.0)()); + } + } +} + +struct CodegenCx<'tcx, M: Module> { + tcx: TyCtxt<'tcx>, + module: M, + global_asm: String, + constants_cx: ConstantCx, + cached_context: Context, + vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>, + debug_context: Option<DebugContext<'tcx>>, + unwind_context: UnwindContext<'tcx>, +} + +impl<'tcx, M: Module> CodegenCx<'tcx, M> { + fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self { + let unwind_context = UnwindContext::new(tcx, module.isa()); + let debug_context = if debug_info { + Some(DebugContext::new(tcx, module.isa())) + } else { + None + }; + CodegenCx { + tcx, + module, + global_asm: String::new(), + constants_cx: ConstantCx::default(), + cached_context: Context::new(), + vtables: FxHashMap::default(), + debug_context, + unwind_context, + } + } + + fn finalize(mut self) -> (M, String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) { + self.constants_cx.finalize(self.tcx, &mut self.module); + ( + self.module, + self.global_asm, + self.debug_context, + self.unwind_context, + ) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct BackendConfig { + pub use_jit: bool, +} + +pub struct CraneliftCodegenBackend { + pub config: BackendConfig, +} + +impl CodegenBackend for CraneliftCodegenBackend { + fn init(&self, sess: &Session) { + if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode { + sess.warn("LTO is not supported. You may get a linker error."); + } + } + + fn metadata_loader(&self) -> Box<dyn MetadataLoader + Sync> { + Box::new(crate::metadata::CraneliftMetadataLoader) + } + + fn provide(&self, _providers: &mut Providers) {} + fn provide_extern(&self, _providers: &mut Providers) {} + + fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> { + vec![] + } + + fn codegen_crate<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box<dyn Any> { + let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config); + + rustc_symbol_mangling::test::report_symbol_names(tcx); + + res + } + + fn join_codegen( + &self, + ongoing_codegen: Box<dyn Any>, + _sess: &Session, + ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported> { + Ok(*ongoing_codegen + .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>() + .unwrap()) + } + + fn link( + &self, + sess: &Session, + codegen_results: CodegenResults, + outputs: &OutputFilenames, + ) -> Result<(), ErrorReported> { + use rustc_codegen_ssa::back::link::link_binary; + + let _timer = sess.prof.generic_activity("link_crate"); + + sess.time("linking", || { + let target_cpu = crate::target_triple(sess).to_string(); + link_binary::<crate::archive::ArArchiveBuilder<'_>>( + sess, + &codegen_results, + outputs, + &codegen_results.crate_name.as_str(), + &target_cpu, + ); + }); + + Ok(()) + } +} + +fn target_triple(sess: &Session) -> target_lexicon::Triple { + sess.target.llvm_target.parse().unwrap() +} + +fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'static> { + use target_lexicon::BinaryFormat; + + let target_triple = crate::target_triple(sess); + + let mut flags_builder = settings::builder(); + if enable_pic { + flags_builder.enable("is_pic").unwrap(); + } else { + flags_builder.set("is_pic", "false").unwrap(); + } + flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided + flags_builder + .set( + "enable_verifier", + if cfg!(debug_assertions) { + "true" + } else { + "false" + }, + ) + .unwrap(); + + let tls_model = match target_triple.binary_format { + BinaryFormat::Elf => "elf_gd", + BinaryFormat::Macho => "macho", + BinaryFormat::Coff => "coff", + _ => "none", + }; + flags_builder.set("tls_model", tls_model).unwrap(); + + flags_builder.set("enable_simd", "true").unwrap(); + + // FIXME(CraneStation/cranelift#732) fix LICM in presence of jump tables + /* + use rustc_session::config::OptLevel; + match sess.opts.optimize { + OptLevel::No => { + flags_builder.set("opt_level", "none").unwrap(); + } + OptLevel::Less | OptLevel::Default => {} + OptLevel::Aggressive => { + flags_builder.set("opt_level", "speed_and_size").unwrap(); + } + OptLevel::Size | OptLevel::SizeMin => { + sess.warn("Optimizing for size is not supported. Just ignoring the request"); + } + }*/ + + let flags = settings::Flags::new(flags_builder); + + let mut isa_builder = cranelift_codegen::isa::lookup(target_triple).unwrap(); + // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt` + // is interpreted as `bsr`. + isa_builder.enable("nehalem").unwrap(); + isa_builder.finish(flags) +} + +/// This is the entrypoint for a hot plugged rustc_codegen_cranelift +#[no_mangle] +pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> { + Box::new(CraneliftCodegenBackend { + config: BackendConfig { use_jit: false }, + }) +} diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs new file mode 100644 index 00000000000..fe5d1d64443 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -0,0 +1,35 @@ +use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; + +use crate::prelude::*; + +pub(crate) fn get_clif_linkage( + mono_item: MonoItem<'_>, + linkage: RLinkage, + visibility: Visibility, +) -> Linkage { + match (linkage, visibility) { + (RLinkage::External, Visibility::Default) => Linkage::Export, + (RLinkage::Internal, Visibility::Default) => Linkage::Local, + (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), + } +} + +pub(crate) fn get_static_linkage(tcx: TyCtxt<'_>, def_id: DefId) -> Linkage { + let fn_attrs = tcx.codegen_fn_attrs(def_id); + + if let Some(linkage) = fn_attrs.linkage { + match linkage { + RLinkage::External => Linkage::Export, + RLinkage::Internal => Linkage::Local, + RLinkage::ExternalWeak | RLinkage::WeakAny => Linkage::Preemptible, + _ => panic!("{:?}", linkage), + } + } else { + if tcx.is_reachable_non_generic(def_id) { + Linkage::Export + } else { + Linkage::Hidden + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs new file mode 100644 index 00000000000..db34d89fe2b --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -0,0 +1,130 @@ +use rustc_hir::LangItem; +use rustc_session::config::EntryFnType; + +use crate::prelude::*; + +/// Create the `main` function which will initialize the rust runtime and call +/// users main function. +pub(crate) fn maybe_create_entry_wrapper( + tcx: TyCtxt<'_>, + module: &mut impl Module, + unwind_context: &mut UnwindContext<'_>, + use_jit: bool, +) { + let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + Some((def_id, entry_ty)) => ( + def_id.to_def_id(), + match entry_ty { + EntryFnType::Main => true, + EntryFnType::Start => false, + }, + ), + None => return, + }; + + let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); + if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + return; + } + + create_entry_fn( + tcx, + module, + unwind_context, + main_def_id, + use_start_lang_item, + use_jit, + ); + + fn create_entry_fn( + tcx: TyCtxt<'_>, + m: &mut impl Module, + unwind_context: &mut UnwindContext<'_>, + rust_main_def_id: DefId, + use_start_lang_item: bool, + use_jit: bool, + ) { + let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); + // Given that `main()` has no arguments, + // then its return type cannot have + // late-bound regions, since late-bound + // regions must appear in the argument + // listing. + let main_ret_ty = tcx.erase_regions(&main_ret_ty.no_bound_vars().unwrap()); + + let cmain_sig = Signature { + params: vec![ + AbiParam::new(m.target_config().pointer_type()), + AbiParam::new(m.target_config().pointer_type()), + ], + returns: vec![AbiParam::new( + m.target_config().pointer_type(), /*isize*/ + )], + call_conv: CallConv::triple_default(m.isa().triple()), + }; + + let cmain_func_id = m + .declare_function("main", Linkage::Export, &cmain_sig) + .unwrap(); + + let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); + + let (main_name, main_sig) = + get_function_name_and_sig(tcx, m.isa().triple(), instance, false); + let main_func_id = m + .declare_function(&main_name, Linkage::Import, &main_sig) + .unwrap(); + + let mut ctx = Context::new(); + ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig.clone()); + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type()); + let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type()); + + crate::atomic_shim::init_global_lock(m, &mut bcx, use_jit); + + let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); + + let call_inst = if use_start_lang_item { + let start_def_id = tcx.require_lang_item(LangItem::Start, None); + let start_instance = Instance::resolve( + tcx, + ParamEnv::reveal_all(), + start_def_id, + tcx.intern_substs(&[main_ret_ty.into()]), + ) + .unwrap() + .unwrap() + .polymorphize(tcx); + let start_func_id = import_function(tcx, m, start_instance); + + let main_val = bcx + .ins() + .func_addr(m.target_config().pointer_type(), main_func_ref); + + let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); + bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + } else { + // using user-defined start fn + bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + }; + + let result = bcx.inst_results(call_inst)[0]; + bcx.ins().return_(&[result]); + bcx.seal_all_blocks(); + bcx.finalize(); + } + m.define_function( + cmain_func_id, + &mut ctx, + &mut cranelift_codegen::binemit::NullTrapSink {}, + ) + .unwrap(); + unwind_context.add_function(cmain_func_id, &ctx, m.isa()); + } +} diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs new file mode 100644 index 00000000000..04369bf89fd --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/metadata.rs @@ -0,0 +1,108 @@ +//! Reading and writing of the rustc metadata for rlibs and dylibs + +use std::convert::TryFrom; +use std::fs::File; +use std::path::Path; + +use rustc_codegen_ssa::METADATA_FILENAME; +use rustc_data_structures::owning_ref::OwningRef; +use rustc_data_structures::rustc_erase_owner; +use rustc_data_structures::sync::MetadataRef; +use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; +use rustc_middle::ty::TyCtxt; +use rustc_session::config; +use rustc_target::spec::Target; + +use crate::backend::WriteMetadata; + +pub(crate) struct CraneliftMetadataLoader; + +impl MetadataLoader for CraneliftMetadataLoader { + fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> { + let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?); + // Iterate over all entries in the archive: + while let Some(entry_result) = archive.next_entry() { + let mut entry = entry_result.map_err(|e| format!("{:?}", e))?; + if entry.header().identifier() == METADATA_FILENAME.as_bytes() { + let mut buf = Vec::with_capacity( + usize::try_from(entry.header().size()) + .expect("Rlib metadata file too big to load into memory."), + ); + ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?; + let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf).into(); + return Ok(rustc_erase_owner!(buf.map_owner_box())); + } + } + + Err("couldn't find metadata entry".to_string()) + } + + fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> { + use object::{Object, ObjectSection}; + let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?; + let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?; + let buf = file + .section_by_name(".rustc") + .ok_or("no .rustc section")? + .data() + .map_err(|e| format!("failed to read .rustc section: {:?}", e))? + .to_owned(); + let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf).into(); + Ok(rustc_erase_owner!(buf.map_owner_box())) + } +} + +// Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 +pub(crate) fn write_metadata<P: WriteMetadata>( + tcx: TyCtxt<'_>, + product: &mut P, +) -> EncodedMetadata { + use snap::write::FrameEncoder; + use std::io::Write; + + #[derive(PartialEq, Eq, PartialOrd, Ord)] + enum MetadataKind { + None, + Uncompressed, + Compressed, + } + + let kind = tcx + .sess + .crate_types() + .iter() + .map(|ty| match *ty { + config::CrateType::Executable + | config::CrateType::Staticlib + | config::CrateType::Cdylib => MetadataKind::None, + + config::CrateType::Rlib => MetadataKind::Uncompressed, + + config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed, + }) + .max() + .unwrap_or(MetadataKind::None); + + if kind == MetadataKind::None { + return EncodedMetadata::new(); + } + + let metadata = tcx.encode_metadata(); + if kind == MetadataKind::Uncompressed { + return metadata; + } + + assert!(kind == MetadataKind::Compressed); + let mut compressed = tcx.metadata_encoding_version(); + FrameEncoder::new(&mut compressed) + .write_all(&metadata.raw_data) + .unwrap(); + + product.add_rustc_section( + rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), + compressed, + tcx.sess.target.options.is_like_osx, + ); + + metadata +} diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs new file mode 100644 index 00000000000..b37826d71f4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -0,0 +1,475 @@ +//! Various operations on integer and floating-point numbers + +use crate::prelude::*; + +pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> { + use BinOp::*; + use IntCC::*; + Some(match bin_op { + Eq => Equal, + Lt => { + if signed { + SignedLessThan + } else { + UnsignedLessThan + } + } + Le => { + if signed { + SignedLessThanOrEqual + } else { + UnsignedLessThanOrEqual + } + } + Ne => NotEqual, + Ge => { + if signed { + SignedGreaterThanOrEqual + } else { + UnsignedGreaterThanOrEqual + } + } + Gt => { + if signed { + SignedGreaterThan + } else { + UnsignedGreaterThan + } + } + _ => return None, + }) +} + +fn codegen_compare_bin_op<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + signed: bool, + lhs: Value, + rhs: Value, +) -> CValue<'tcx> { + let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap(); + let val = fx.bcx.ins().icmp(intcc, lhs, rhs); + let val = fx.bcx.ins().bint(types::I8, val); + CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)) +} + +pub(crate) fn codegen_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + match bin_op { + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + match in_lhs.layout().ty.kind() { + ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => { + let signed = type_sign(in_lhs.layout().ty); + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let (lhs, rhs) = if (bin_op == BinOp::Eq || bin_op == BinOp::Ne) + && (in_lhs.layout().ty.kind() == fx.tcx.types.i8.kind() + || in_lhs.layout().ty.kind() == fx.tcx.types.i16.kind()) + { + // FIXME(CraneStation/cranelift#896) icmp_imm.i8/i16 with eq/ne for signed ints is implemented wrong. + ( + fx.bcx.ins().sextend(types::I32, lhs), + fx.bcx.ins().sextend(types::I32, rhs), + ) + } else { + (lhs, rhs) + }; + + return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs); + } + _ => {} + } + } + _ => {} + } + + match in_lhs.layout().ty.kind() { + ty::Bool => crate::num::trans_bool_binop(fx, bin_op, in_lhs, in_rhs), + ty::Uint(_) | ty::Int(_) => crate::num::trans_int_binop(fx, bin_op, in_lhs, in_rhs), + ty::Float(_) => crate::num::trans_float_binop(fx, bin_op, in_lhs, in_rhs), + ty::RawPtr(..) | ty::FnPtr(..) => crate::num::trans_ptr_binop(fx, bin_op, in_lhs, in_rhs), + _ => unreachable!( + "{:?}({:?}, {:?})", + bin_op, + in_lhs.layout().ty, + in_rhs.layout().ty + ), + } +} + +pub(crate) fn trans_bool_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + let res = match bin_op { + BinOp::BitXor => b.bxor(lhs, rhs), + BinOp::BitAnd => b.band(lhs, rhs), + BinOp::BitOr => b.bor(lhs, rhs), + // Compare binops handles by `codegen_binop`. + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + }; + + CValue::by_val(res, fx.layout_of(fx.tcx.types.bool)) +} + +pub(crate) fn trans_int_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + if bin_op != BinOp::Shl && bin_op != BinOp::Shr { + assert_eq!( + in_lhs.layout().ty, + in_rhs.layout().ty, + "int binop requires lhs and rhs of same type" + ); + } + + if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, false, in_lhs, in_rhs) { + return res; + } + + let signed = type_sign(in_lhs.layout().ty); + + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + let val = match bin_op { + BinOp::Add => b.iadd(lhs, rhs), + BinOp::Sub => b.isub(lhs, rhs), + BinOp::Mul => b.imul(lhs, rhs), + BinOp::Div => { + if signed { + b.sdiv(lhs, rhs) + } else { + b.udiv(lhs, rhs) + } + } + BinOp::Rem => { + if signed { + b.srem(lhs, rhs) + } else { + b.urem(lhs, rhs) + } + } + BinOp::BitXor => b.bxor(lhs, rhs), + BinOp::BitAnd => b.band(lhs, rhs), + BinOp::BitOr => b.bor(lhs, rhs), + BinOp::Shl => { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); + let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); + fx.bcx.ins().ishl(lhs, actual_shift) + } + BinOp::Shr => { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); + let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); + if signed { + fx.bcx.ins().sshr(lhs, actual_shift) + } else { + fx.bcx.ins().ushr(lhs, actual_shift) + } + } + // Compare binops handles by `codegen_binop`. + _ => unreachable!( + "{:?}({:?}, {:?})", + bin_op, + in_lhs.layout().ty, + in_rhs.layout().ty + ), + }; + + CValue::by_val(val, in_lhs.layout()) +} + +pub(crate) fn trans_checked_int_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + if bin_op != BinOp::Shl && bin_op != BinOp::Shr { + assert_eq!( + in_lhs.layout().ty, + in_rhs.layout().ty, + "checked int binop requires lhs and rhs of same type" + ); + } + + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, true, in_lhs, in_rhs) { + return res; + } + + let signed = type_sign(in_lhs.layout().ty); + + let (res, has_overflow) = match bin_op { + BinOp::Add => { + /*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs); + (val, c_out)*/ + // FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16 + let val = fx.bcx.ins().iadd(lhs, rhs); + let has_overflow = if !signed { + fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs) + } else { + let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs); + fx.bcx.ins().bxor(rhs_is_negative, slt) + }; + (val, has_overflow) + } + BinOp::Sub => { + /*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs); + (val, b_out)*/ + // FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16 + let val = fx.bcx.ins().isub(lhs, rhs); + let has_overflow = if !signed { + fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs) + } else { + let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0); + let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs); + fx.bcx.ins().bxor(rhs_is_negative, sgt) + }; + (val, has_overflow) + } + BinOp::Mul => { + let ty = fx.bcx.func.dfg.value_type(lhs); + match ty { + types::I8 | types::I16 | types::I32 if !signed => { + let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs); + let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs); + let val = fx.bcx.ins().imul(lhs, rhs); + let has_overflow = fx.bcx.ins().icmp_imm( + IntCC::UnsignedGreaterThan, + val, + (1 << ty.bits()) - 1, + ); + let val = fx.bcx.ins().ireduce(ty, val); + (val, has_overflow) + } + types::I8 | types::I16 | types::I32 if signed => { + let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs); + let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs); + let val = fx.bcx.ins().imul(lhs, rhs); + let has_underflow = + fx.bcx + .ins() + .icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1))); + let has_overflow = fx.bcx.ins().icmp_imm( + IntCC::SignedGreaterThan, + val, + (1 << (ty.bits() - 1)) - 1, + ); + let val = fx.bcx.ins().ireduce(ty, val); + (val, fx.bcx.ins().bor(has_underflow, has_overflow)) + } + types::I64 => { + //let val = fx.easy_call("__mulodi4", &[lhs, rhs, overflow_ptr], types::I64); + let val = fx.bcx.ins().imul(lhs, rhs); + let has_overflow = if !signed { + let val_hi = fx.bcx.ins().umulhi(lhs, rhs); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) + } else { + let val_hi = fx.bcx.ins().smulhi(lhs, rhs); + let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0); + let not_all_ones = fx.bcx.ins().icmp_imm( + IntCC::NotEqual, + val_hi, + u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64, + ); + fx.bcx.ins().band(not_all_zero, not_all_ones) + }; + (val, has_overflow) + } + types::I128 => { + unreachable!("i128 should have been handled by codegen_i128::maybe_codegen") + } + _ => unreachable!("invalid non-integer type {}", ty), + } + } + BinOp::Shl => { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); + let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); + let val = fx.bcx.ins().ishl(lhs, actual_shift); + let ty = fx.bcx.func.dfg.value_type(val); + let max_shift = i64::from(ty.bits()) - 1; + let has_overflow = fx + .bcx + .ins() + .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift); + (val, has_overflow) + } + BinOp::Shr => { + let lhs_ty = fx.bcx.func.dfg.value_type(lhs); + let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); + let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); + let val = if !signed { + fx.bcx.ins().ushr(lhs, actual_shift) + } else { + fx.bcx.ins().sshr(lhs, actual_shift) + }; + let ty = fx.bcx.func.dfg.value_type(val); + let max_shift = i64::from(ty.bits()) - 1; + let has_overflow = fx + .bcx + .ins() + .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift); + (val, has_overflow) + } + _ => bug!( + "binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", + bin_op, + in_lhs, + in_rhs + ), + }; + + let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow); + + // FIXME directly write to result place instead + let out_place = CPlace::new_stack_slot( + fx, + fx.layout_of( + fx.tcx + .mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()), + ), + ); + let out_layout = out_place.layout(); + out_place.write_cvalue(fx, CValue::by_val_pair(res, has_overflow, out_layout)); + + out_place.to_cvalue(fx) +} + +pub(crate) fn trans_float_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty); + + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + let b = fx.bcx.ins(); + let res = match bin_op { + BinOp::Add => b.fadd(lhs, rhs), + BinOp::Sub => b.fsub(lhs, rhs), + BinOp::Mul => b.fmul(lhs, rhs), + BinOp::Div => b.fdiv(lhs, rhs), + BinOp::Rem => { + let name = match in_lhs.layout().ty.kind() { + ty::Float(FloatTy::F32) => "fmodf", + ty::Float(FloatTy::F64) => "fmod", + _ => bug!(), + }; + return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty); + } + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + let fltcc = match bin_op { + BinOp::Eq => FloatCC::Equal, + BinOp::Lt => FloatCC::LessThan, + BinOp::Le => FloatCC::LessThanOrEqual, + BinOp::Ne => FloatCC::NotEqual, + BinOp::Ge => FloatCC::GreaterThanOrEqual, + BinOp::Gt => FloatCC::GreaterThan, + _ => unreachable!(), + }; + let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs); + let val = fx.bcx.ins().bint(types::I8, val); + return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)); + } + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + }; + + CValue::by_val(res, in_lhs.layout()) +} + +pub(crate) fn trans_ptr_binop<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + bin_op: BinOp, + in_lhs: CValue<'tcx>, + in_rhs: CValue<'tcx>, +) -> CValue<'tcx> { + let is_thin_ptr = in_lhs + .layout() + .ty + .builtin_deref(true) + .map(|TypeAndMut { ty, mutbl: _ }| !has_ptr_meta(fx.tcx, ty)) + .unwrap_or(true); + + if is_thin_ptr { + match bin_op { + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + let lhs = in_lhs.load_scalar(fx); + let rhs = in_rhs.load_scalar(fx); + + return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs); + } + BinOp::Offset => { + let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty; + let (base, offset) = (in_lhs, in_rhs.load_scalar(fx)); + let pointee_size = fx.layout_of(pointee_ty).size.bytes(); + let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); + let base_val = base.load_scalar(fx); + let res = fx.bcx.ins().iadd(base_val, ptr_diff); + return CValue::by_val(res, base.layout()); + } + _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), + }; + } else { + let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx); + let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx); + + let res = match bin_op { + BinOp::Eq => { + let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr); + let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra); + fx.bcx.ins().band(ptr_eq, extra_eq) + } + BinOp::Ne => { + let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr); + let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra); + fx.bcx.ins().bor(ptr_ne, extra_ne) + } + BinOp::Lt | BinOp::Le | BinOp::Ge | BinOp::Gt => { + let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr); + + let ptr_cmp = + fx.bcx + .ins() + .icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr); + let extra_cmp = fx.bcx.ins().icmp( + bin_op_to_intcc(bin_op, false).unwrap(), + lhs_extra, + rhs_extra, + ); + + fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp) + } + _ => panic!("bin_op {:?} on ptr", bin_op), + }; + + CValue::by_val( + fx.bcx.ins().bint(types::I8, res), + fx.layout_of(fx.tcx.types.bool), + ) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs new file mode 100644 index 00000000000..f02732014d1 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs @@ -0,0 +1,40 @@ +//! This optimization moves cold code to the end of the function. +//! +//! Some code is executed much less often than other code. For example panicking or the +//! landingpads for unwinding. By moving this cold code to the end of the function the average +//! amount of jumps is reduced and the code locality is improved. +//! +//! # Undefined behaviour +//! +//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself. + +use crate::prelude::*; + +pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block>) { + // FIXME Move the block in place instead of remove and append once + // bytecodealliance/cranelift#1339 is implemented. + + let mut block_insts = FxHashMap::default(); + for block in cold_blocks + .keys() + .filter(|&block| cold_blocks.contains(block)) + { + let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>(); + for &inst in &insts { + ctx.func.layout.remove_inst(inst); + } + block_insts.insert(block, insts); + ctx.func.layout.remove_block(block); + } + + // And then append them at the back again. + for block in cold_blocks + .keys() + .filter(|&block| cold_blocks.contains(block)) + { + ctx.func.layout.append_block(block); + for inst in block_insts.remove(&block).unwrap() { + ctx.func.layout.append_inst(inst, block); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs new file mode 100644 index 00000000000..3ce7f8cd9a8 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs @@ -0,0 +1,25 @@ +//! Various optimizations specific to cg_clif + +use crate::prelude::*; + +mod code_layout; +pub(crate) mod peephole; +mod stack2reg; + +pub(crate) fn optimize_function<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + ctx: &mut Context, + cold_blocks: &EntitySet<Block>, + clif_comments: &mut crate::pretty_clif::CommentWriter, +) { + // The code_layout optimization is very cheap. + self::code_layout::optimize_function(ctx, cold_blocks); + + if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { + return; // FIXME classify optimizations over opt levels + } + self::stack2reg::optimize_function(ctx, clif_comments); + crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments); + crate::base::verify_func(tcx, &*clif_comments, &ctx.func); +} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs new file mode 100644 index 00000000000..f8e0f3af3d0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs @@ -0,0 +1,83 @@ +//! Peephole optimizations that can be performed while creating clif ir. + +use cranelift_codegen::ir::{ + condcodes::IntCC, types, InstBuilder, InstructionData, Opcode, Value, ValueDef, +}; +use cranelift_frontend::FunctionBuilder; + +/// If the given value was produced by a `bint` instruction, return it's input, otherwise return the +/// given value. +pub(crate) fn maybe_unwrap_bint(bcx: &mut FunctionBuilder<'_>, arg: Value) -> Value { + if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + match bcx.func.dfg[arg_inst] { + InstructionData::Unary { + opcode: Opcode::Bint, + arg, + } => arg, + _ => arg, + } + } else { + arg + } +} + +/// If the given value was produced by the lowering of `Rvalue::Not` return the input and true, +/// otherwise return the given value and false. +pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) { + if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + match bcx.func.dfg[arg_inst] { + // This is the lowering of `Rvalue::Not` + InstructionData::IntCompareImm { + opcode: Opcode::IcmpImm, + cond: IntCC::Equal, + arg, + imm, + } if imm.bits() == 0 => (arg, true), + _ => (arg, false), + } + } else { + (arg, false) + } +} + +pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -> Value { + if bcx.func.dfg.value_type(arg).is_bool() { + return arg; + } + + (|| { + let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) { + arg_inst + } else { + return None; + }; + + match bcx.func.dfg[arg_inst] { + // This is the lowering of Rvalue::Not + InstructionData::Load { + opcode: Opcode::Load, + arg: ptr, + flags, + offset, + } => { + // Using `load.i8 + uextend.i32` would legalize to `uload8 + ireduce.i8 + + // uextend.i32`. Just `uload8` is much faster. + match bcx.func.dfg.ctrl_typevar(arg_inst) { + types::I8 => Some(bcx.ins().uload8(types::I32, flags, ptr, offset)), + types::I16 => Some(bcx.ins().uload16(types::I32, flags, ptr, offset)), + _ => None, + } + } + _ => None, + } + })() + .unwrap_or_else(|| { + match bcx.func.dfg.value_type(arg) { + types::I8 | types::I32 => { + // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented + bcx.ins().uextend(types::I32, arg) + } + _ => arg, + } + }) +} diff --git a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs new file mode 100644 index 00000000000..f368d65f7f8 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs @@ -0,0 +1,507 @@ +//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible. +//! +//! # Undefined behaviour +//! +//! This optimization is based on the assumption that stack slots which don't have their address +//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the +//! function which has the stack slots. This optimization also assumes that stack slot accesses +//! are never out of bounds. If these assumptions are not correct, then this optimization may remove +//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value +//! being loaded by a `stack_load`. + +use std::collections::BTreeMap; +use std::fmt; +use std::ops::Not; + +use rustc_data_structures::fx::FxHashSet; + +use cranelift_codegen::cursor::{Cursor, FuncCursor}; +use cranelift_codegen::ir::immediates::Offset32; +use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; + +use crate::prelude::*; + +/// Workaround for `StackSlot` not implementing `Ord`. +#[derive(Copy, Clone, PartialEq, Eq)] +struct OrdStackSlot(StackSlot); + +impl fmt::Debug for OrdStackSlot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.0) + } +} + +impl PartialOrd for OrdStackSlot { + fn partial_cmp(&self, rhs: &Self) -> Option<std::cmp::Ordering> { + self.0.as_u32().partial_cmp(&rhs.0.as_u32()) + } +} + +impl Ord for OrdStackSlot { + fn cmp(&self, rhs: &Self) -> std::cmp::Ordering { + self.0.as_u32().cmp(&rhs.0.as_u32()) + } +} + +#[derive(Debug, Default)] +struct StackSlotUsage { + stack_addr: FxHashSet<Inst>, + stack_load: FxHashSet<Inst>, + stack_store: FxHashSet<Inst>, +} + +impl StackSlotUsage { + fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec<Inst> { + self.stack_store + .iter() + .cloned() + .filter(|&store| { + match spatial_overlap(&ctx.func, store, load) { + SpatialOverlap::No => false, // Can never be the source of the loaded value. + SpatialOverlap::Partial | SpatialOverlap::Full => true, + } + }) + .filter(|&store| { + match temporal_order(ctx, store, load) { + TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. + TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, + } + }) + .collect::<Vec<Inst>>() + } + + fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec<Inst> { + self.stack_load + .iter() + .cloned() + .filter(|&load| { + match spatial_overlap(&ctx.func, store, load) { + SpatialOverlap::No => false, // Can never be the source of the loaded value. + SpatialOverlap::Partial | SpatialOverlap::Full => true, + } + }) + .filter(|&load| { + match temporal_order(ctx, store, load) { + TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. + TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, + } + }) + .collect::<Vec<Inst>>() + } + + fn remove_unused_stack_addr(func: &mut Function, inst: Inst) { + func.dfg.detach_results(inst); + func.dfg.replace(inst).nop(); + } + + fn remove_unused_load(func: &mut Function, load: Inst) { + func.dfg.detach_results(load); + func.dfg.replace(load).nop(); + } + + fn remove_dead_store(&mut self, func: &mut Function, store: Inst) { + func.dfg.replace(store).nop(); + self.stack_store.remove(&store); + } + + fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) { + let loaded_value = func.dfg.inst_results(load)[0]; + let loaded_type = func.dfg.value_type(loaded_value); + + if func.dfg.value_type(value) == loaded_type { + func.dfg.detach_results(load); + func.dfg.replace(load).nop(); + func.dfg.change_to_alias(loaded_value, value); + } else { + func.dfg.replace(load).bitcast(loaded_type, value); + } + + self.stack_load.remove(&load); + } +} + +struct OptimizeContext<'a> { + ctx: &'a mut Context, + stack_slot_usage_map: BTreeMap<OrdStackSlot, StackSlotUsage>, +} + +impl<'a> OptimizeContext<'a> { + fn for_context(ctx: &'a mut Context) -> Self { + ctx.flowgraph(); // Compute cfg and domtree. + + // Record all stack_addr, stack_load and stack_store instructions. + let mut stack_slot_usage_map = BTreeMap::<OrdStackSlot, StackSlotUsage>::new(); + + let mut cursor = FuncCursor::new(&mut ctx.func); + while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + match cursor.func.dfg[inst] { + InstructionData::StackLoad { + opcode: Opcode::StackAddr, + stack_slot, + offset: _, + } => { + stack_slot_usage_map + .entry(OrdStackSlot(stack_slot)) + .or_insert_with(StackSlotUsage::default) + .stack_addr + .insert(inst); + } + InstructionData::StackLoad { + opcode: Opcode::StackLoad, + stack_slot, + offset: _, + } => { + stack_slot_usage_map + .entry(OrdStackSlot(stack_slot)) + .or_insert_with(StackSlotUsage::default) + .stack_load + .insert(inst); + } + InstructionData::StackStore { + opcode: Opcode::StackStore, + arg: _, + stack_slot, + offset: _, + } => { + stack_slot_usage_map + .entry(OrdStackSlot(stack_slot)) + .or_insert_with(StackSlotUsage::default) + .stack_store + .insert(inst); + } + _ => {} + } + } + } + + OptimizeContext { + ctx, + stack_slot_usage_map, + } + } +} + +pub(super) fn optimize_function( + ctx: &mut Context, + #[cfg_attr(not(debug_assertions), allow(unused_variables))] clif_comments: &mut crate::pretty_clif::CommentWriter, +) { + combine_stack_addr_with_load_store(&mut ctx.func); + + let mut opt_ctx = OptimizeContext::for_context(ctx); + + // FIXME Repeat following instructions until fixpoint. + + remove_unused_stack_addr_and_stack_load(&mut opt_ctx); + + #[cfg(debug_assertions)] + { + for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { + clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); + } + } + + for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() { + if users.stack_addr.is_empty().not() { + // Stack addr leaked; there may be unknown loads and stores. + // FIXME use stacked borrows to optimize + continue; + } + + for load in users.stack_load.clone().into_iter() { + let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); + + #[cfg(debug_assertions)] + for &store in &potential_stores { + clif_comments.add_comment( + load, + format!( + "Potential store -> load forwarding {} -> {} ({:?}, {:?})", + opt_ctx.ctx.func.dfg.display_inst(store, None), + opt_ctx.ctx.func.dfg.display_inst(load, None), + spatial_overlap(&opt_ctx.ctx.func, store, load), + temporal_order(&opt_ctx.ctx, store, load), + ), + ); + } + + match *potential_stores { + [] => { + #[cfg(debug_assertions)] + clif_comments.add_comment(load, format!("[BUG?] Reading uninitialized memory")); + } + [store] + if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full + && temporal_order(&opt_ctx.ctx, store, load) + == TemporalOrder::DefinitivelyBefore => + { + // Only one store could have been the origin of the value. + let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; + + #[cfg(debug_assertions)] + clif_comments + .add_comment(load, format!("Store to load forward {} -> {}", store, load)); + + users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); + } + _ => {} // FIXME implement this + } + } + + for store in users.stack_store.clone().into_iter() { + let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); + + #[cfg(debug_assertions)] + for &load in &potential_loads { + clif_comments.add_comment( + store, + format!( + "Potential load from store {} <- {} ({:?}, {:?})", + opt_ctx.ctx.func.dfg.display_inst(load, None), + opt_ctx.ctx.func.dfg.display_inst(store, None), + spatial_overlap(&opt_ctx.ctx.func, store, load), + temporal_order(&opt_ctx.ctx, store, load), + ), + ); + } + + if potential_loads.is_empty() { + // Never loaded; can safely remove all stores and the stack slot. + // FIXME also remove stores when there is always a next store before a load. + + #[cfg(debug_assertions)] + clif_comments.add_comment( + store, + format!( + "Remove dead stack store {} of {}", + opt_ctx.ctx.func.dfg.display_inst(store, None), + stack_slot.0 + ), + ); + + users.remove_dead_store(&mut opt_ctx.ctx.func, store); + } + } + + if users.stack_store.is_empty() && users.stack_load.is_empty() { + opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0; + } + } +} + +fn combine_stack_addr_with_load_store(func: &mut Function) { + // Turn load and store into stack_load and stack_store when possible. + let mut cursor = FuncCursor::new(func); + while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + match cursor.func.dfg[inst] { + InstructionData::Load { + opcode: Opcode::Load, + arg: addr, + flags: _, + offset, + } => { + if cursor.func.dfg.ctrl_typevar(inst) == types::I128 + || cursor.func.dfg.ctrl_typevar(inst).is_vector() + { + continue; // WORKAROUD: stack_load.i128 not yet implemented + } + if let Some((stack_slot, stack_addr_offset)) = + try_get_stack_slot_and_offset_for_addr(cursor.func, addr) + { + if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) + { + let ty = cursor.func.dfg.ctrl_typevar(inst); + cursor.func.dfg.replace(inst).stack_load( + ty, + stack_slot, + combined_offset, + ); + } + } + } + InstructionData::Store { + opcode: Opcode::Store, + args: [value, addr], + flags: _, + offset, + } => { + if cursor.func.dfg.ctrl_typevar(inst) == types::I128 + || cursor.func.dfg.ctrl_typevar(inst).is_vector() + { + continue; // WORKAROUND: stack_store.i128 not yet implemented + } + if let Some((stack_slot, stack_addr_offset)) = + try_get_stack_slot_and_offset_for_addr(cursor.func, addr) + { + if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) + { + cursor.func.dfg.replace(inst).stack_store( + value, + stack_slot, + combined_offset, + ); + } + } + } + _ => {} + } + } + } +} + +fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) { + // FIXME incrementally rebuild on each call? + let mut stack_addr_load_insts_users = FxHashMap::<Inst, FxHashSet<Inst>>::default(); + + let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func); + while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + for &arg in cursor.func.dfg.inst_args(inst) { + if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) { + match cursor.func.dfg[arg_origin].opcode() { + Opcode::StackAddr | Opcode::StackLoad => { + stack_addr_load_insts_users + .entry(arg_origin) + .or_insert_with(FxHashSet::default) + .insert(inst); + } + _ => {} + } + } + } + } + } + + #[cfg(debug_assertions)] + for inst in stack_addr_load_insts_users.keys() { + let mut is_recorded_stack_addr_or_stack_load = false; + for stack_slot_users in opt_ctx.stack_slot_usage_map.values() { + is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst) + || stack_slot_users.stack_load.contains(inst); + } + assert!(is_recorded_stack_addr_or_stack_load); + } + + // Replace all unused stack_addr and stack_load instructions with nop. + let mut func = &mut opt_ctx.ctx.func; + + for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() { + stack_slot_users + .stack_addr + .drain_filter(|inst| { + stack_addr_load_insts_users + .get(inst) + .map(|users| users.is_empty()) + .unwrap_or(true) + }) + .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst)); + + stack_slot_users + .stack_load + .drain_filter(|inst| { + stack_addr_load_insts_users + .get(inst) + .map(|users| users.is_empty()) + .unwrap_or(true) + }) + .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst)); + } +} + +fn try_get_stack_slot_and_offset_for_addr( + func: &Function, + addr: Value, +) -> Option<(StackSlot, Offset32)> { + if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) { + if let InstructionData::StackLoad { + opcode: Opcode::StackAddr, + stack_slot, + offset, + } = func.dfg[addr_inst] + { + return Some((stack_slot, offset)); + } + } + None +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum SpatialOverlap { + No, + Partial, + Full, +} + +fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap { + fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) { + match func.dfg[inst] { + InstructionData::StackLoad { + opcode: Opcode::StackAddr, + stack_slot, + offset, + } + | InstructionData::StackLoad { + opcode: Opcode::StackLoad, + stack_slot, + offset, + } + | InstructionData::StackStore { + opcode: Opcode::StackStore, + stack_slot, + offset, + arg: _, + } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()), + _ => unreachable!("{:?}", func.dfg[inst]), + } + } + + debug_assert_ne!(src, dest); + + let (src_ss, src_offset, src_size) = inst_info(func, src); + let (dest_ss, dest_offset, dest_size) = inst_info(func, dest); + + if src_ss != dest_ss { + return SpatialOverlap::No; + } + + if src_offset == dest_offset && src_size == dest_size { + return SpatialOverlap::Full; + } + + let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into(); + let dest_end: i64 = dest_offset + .try_add_i64(i64::from(dest_size)) + .unwrap() + .into(); + if src_end <= dest_offset.into() || dest_end <= src_offset.into() { + return SpatialOverlap::No; + } + + SpatialOverlap::Partial +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum TemporalOrder { + /// `src` will never be executed before `dest`. + NeverBefore, + + /// `src` may be executed before `dest`. + MaybeBefore, + + /// `src` will always be executed before `dest`. + /// There may still be other instructions in between. + DefinitivelyBefore, +} + +fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder { + debug_assert_ne!(src, dest); + + if ctx.domtree.dominates(src, dest, &ctx.func.layout) { + TemporalOrder::DefinitivelyBefore + } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) { + TemporalOrder::NeverBefore + } else { + TemporalOrder::MaybeBefore + } +} diff --git a/compiler/rustc_codegen_cranelift/src/pointer.rs b/compiler/rustc_codegen_cranelift/src/pointer.rs new file mode 100644 index 00000000000..b2036d7bcd4 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/pointer.rs @@ -0,0 +1,206 @@ +//! Defines [`Pointer`] which is used to improve the quality of the generated clif ir for pointer +//! operations. + +use crate::prelude::*; + +use rustc_target::abi::Align; + +use cranelift_codegen::ir::immediates::Offset32; + +/// A pointer pointing either to a certain address, a certain stack slot or nothing. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Pointer { + base: PointerBase, + offset: Offset32, +} + +#[derive(Copy, Clone, Debug)] +pub(crate) enum PointerBase { + Addr(Value), + Stack(StackSlot), + Dangling(Align), +} + +impl Pointer { + pub(crate) fn new(addr: Value) -> Self { + Pointer { + base: PointerBase::Addr(addr), + offset: Offset32::new(0), + } + } + + pub(crate) fn stack_slot(stack_slot: StackSlot) -> Self { + Pointer { + base: PointerBase::Stack(stack_slot), + offset: Offset32::new(0), + } + } + + pub(crate) fn const_addr<'a, 'tcx>( + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + addr: i64, + ) -> Self { + let addr = fx.bcx.ins().iconst(fx.pointer_type, addr); + Pointer { + base: PointerBase::Addr(addr), + offset: Offset32::new(0), + } + } + + pub(crate) fn dangling(align: Align) -> Self { + Pointer { + base: PointerBase::Dangling(align), + offset: Offset32::new(0), + } + } + + #[cfg(debug_assertions)] + pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) { + (self.base, self.offset) + } + + pub(crate) fn get_addr<'a, 'tcx>(self, fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> Value { + match self.base { + PointerBase::Addr(base_addr) => { + let offset: i64 = self.offset.into(); + if offset == 0 { + base_addr + } else { + fx.bcx.ins().iadd_imm(base_addr, offset) + } + } + PointerBase::Stack(stack_slot) => { + fx.bcx + .ins() + .stack_addr(fx.pointer_type, stack_slot, self.offset) + } + PointerBase::Dangling(align) => fx + .bcx + .ins() + .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()), + } + } + + pub(crate) fn offset<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + extra_offset: Offset32, + ) -> Self { + self.offset_i64(fx, extra_offset.into()) + } + + pub(crate) fn offset_i64<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + extra_offset: i64, + ) -> Self { + if let Some(new_offset) = self.offset.try_add_i64(extra_offset) { + Pointer { + base: self.base, + offset: new_offset, + } + } else { + let base_offset: i64 = self.offset.into(); + if let Some(new_offset) = base_offset.checked_add(extra_offset) { + let base_addr = match self.base { + PointerBase::Addr(addr) => addr, + PointerBase::Stack(stack_slot) => { + fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0) + } + PointerBase::Dangling(align) => fx + .bcx + .ins() + .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()), + }; + let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset); + Pointer { + base: PointerBase::Addr(addr), + offset: Offset32::new(0), + } + } else { + panic!( + "self.offset ({}) + extra_offset ({}) not representable in i64", + base_offset, extra_offset + ); + } + } + } + + pub(crate) fn offset_value<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + extra_offset: Value, + ) -> Self { + match self.base { + PointerBase::Addr(addr) => Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)), + offset: self.offset, + }, + PointerBase::Stack(stack_slot) => { + let base_addr = fx + .bcx + .ins() + .stack_addr(fx.pointer_type, stack_slot, self.offset); + Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(base_addr, extra_offset)), + offset: Offset32::new(0), + } + } + PointerBase::Dangling(align) => { + let addr = fx + .bcx + .ins() + .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()); + Pointer { + base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)), + offset: self.offset, + } + } + } + } + + pub(crate) fn load<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + ty: Type, + flags: MemFlags, + ) -> Value { + match self.base { + PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset), + PointerBase::Stack(stack_slot) => { + if ty == types::I128 || ty.is_vector() { + // WORKAROUND for stack_load.i128 and stack_load.iXxY not being implemented + let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0); + fx.bcx.ins().load(ty, flags, base_addr, self.offset) + } else { + fx.bcx.ins().stack_load(ty, stack_slot, self.offset) + } + } + PointerBase::Dangling(_align) => unreachable!(), + } + } + + pub(crate) fn store<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Module>, + value: Value, + flags: MemFlags, + ) { + match self.base { + PointerBase::Addr(base_addr) => { + fx.bcx.ins().store(flags, value, base_addr, self.offset); + } + PointerBase::Stack(stack_slot) => { + let val_ty = fx.bcx.func.dfg.value_type(value); + if val_ty == types::I128 || val_ty.is_vector() { + // WORKAROUND for stack_store.i128 and stack_store.iXxY not being implemented + let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0); + fx.bcx.ins().store(flags, value, base_addr, self.offset); + } else { + fx.bcx.ins().stack_store(value, stack_slot, self.offset); + } + } + PointerBase::Dangling(_align) => unreachable!(), + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs new file mode 100644 index 00000000000..7f8ab953d71 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -0,0 +1,287 @@ +//! This module provides the [CommentWriter] which makes it possible +//! to add comments to the written cranelift ir. +//! +//! # Example +//! +//! ```clif +//! test compile +//! target x86_64 +//! +//! function u0:0(i64, i64, i64) system_v { +//! ; symbol _ZN119_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$u27$a$u20$$RF$$u27$b$u20$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17he85059d5e6a760a0E +//! ; instance Instance { def: Item(DefId(0/0:29 ~ example[8787]::{{impl}}[0]::call_once[0])), substs: [ReErased, ReErased] } +//! ; sig ([IsNotEmpty, (&&[u16],)]; c_variadic: false)->(u8, u8) +//! +//! ; ssa {_2: NOT_SSA, _4: NOT_SSA, _0: NOT_SSA, _3: (empty), _1: NOT_SSA} +//! ; msg loc.idx param pass mode ssa flags ty +//! ; ret _0 = v0 ByRef NOT_SSA (u8, u8) +//! ; arg _1 = v1 ByRef NOT_SSA IsNotEmpty +//! ; arg _2.0 = v2 ByVal(types::I64) NOT_SSA &&[u16] +//! +//! ss0 = explicit_slot 0 ; _1: IsNotEmpty size=0 align=1,8 +//! ss1 = explicit_slot 8 ; _2: (&&[u16],) size=8 align=8,8 +//! ss2 = explicit_slot 8 ; _4: (&&[u16],) size=8 align=8,8 +//! sig0 = (i64, i64, i64) system_v +//! sig1 = (i64, i64, i64) system_v +//! fn0 = colocated u0:6 sig1 ; Instance { def: Item(DefId(0/0:31 ~ example[8787]::{{impl}}[1]::call_mut[0])), substs: [ReErased, ReErased] } +//! +//! block0(v0: i64, v1: i64, v2: i64): +//! v3 = stack_addr.i64 ss0 +//! v4 = stack_addr.i64 ss1 +//! store v2, v4 +//! v5 = stack_addr.i64 ss2 +//! jump block1 +//! +//! block1: +//! nop +//! ; _3 = &mut _1 +//! ; _4 = _2 +//! v6 = load.i64 v4 +//! store v6, v5 +//! ; +//! ; _0 = const mini_core::FnMut::call_mut(move _3, move _4) +//! v7 = load.i64 v5 +//! call fn0(v0, v3, v7) +//! jump block2 +//! +//! block2: +//! nop +//! ; +//! ; return +//! return +//! } +//! ``` + +use std::fmt; + +use cranelift_codegen::{ + entity::SecondaryMap, + ir::{entities::AnyEntity, function::DisplayFunctionAnnotations}, + write::{FuncWriter, PlainWriter}, +}; + +use rustc_session::config::OutputType; + +use crate::prelude::*; + +#[derive(Debug)] +pub(crate) struct CommentWriter { + global_comments: Vec<String>, + entity_comments: FxHashMap<AnyEntity, String>, +} + +impl CommentWriter { + pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + let global_comments = if cfg!(debug_assertions) { + vec![ + format!("symbol {}", tcx.symbol_name(instance).name), + format!("instance {:?}", instance), + format!( + "sig {:?}", + tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + &crate::abi::fn_sig_for_fn_abi(tcx, instance) + ) + ), + String::new(), + ] + } else { + vec![] + }; + + CommentWriter { + global_comments, + entity_comments: FxHashMap::default(), + } + } +} + +#[cfg(debug_assertions)] +impl CommentWriter { + pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) { + self.global_comments.push(comment.into()); + } + + pub(crate) fn add_comment<S: Into<String> + AsRef<str>, E: Into<AnyEntity>>( + &mut self, + entity: E, + comment: S, + ) { + use std::collections::hash_map::Entry; + match self.entity_comments.entry(entity.into()) { + Entry::Occupied(mut occ) => { + occ.get_mut().push('\n'); + occ.get_mut().push_str(comment.as_ref()); + } + Entry::Vacant(vac) => { + vac.insert(comment.into()); + } + } + } +} + +impl FuncWriter for &'_ CommentWriter { + fn write_preamble( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + reg_info: Option<&isa::RegInfo>, + ) -> Result<bool, fmt::Error> { + for comment in &self.global_comments { + if !comment.is_empty() { + writeln!(w, "; {}", comment)?; + } else { + writeln!(w, "")?; + } + } + if !self.global_comments.is_empty() { + writeln!(w, "")?; + } + + self.super_preamble(w, func, reg_info) + } + + fn write_entity_definition( + &mut self, + w: &mut dyn fmt::Write, + _func: &Function, + entity: AnyEntity, + value: &dyn fmt::Display, + ) -> fmt::Result { + write!(w, " {} = {}", entity, value)?; + + if let Some(comment) = self.entity_comments.get(&entity) { + writeln!(w, " ; {}", comment.replace('\n', "\n; ")) + } else { + writeln!(w, "") + } + } + + fn write_block_header( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + isa: Option<&dyn isa::TargetIsa>, + block: Block, + indent: usize, + ) -> fmt::Result { + PlainWriter.write_block_header(w, func, isa, block, indent) + } + + fn write_instruction( + &mut self, + w: &mut dyn fmt::Write, + func: &Function, + aliases: &SecondaryMap<Value, Vec<Value>>, + isa: Option<&dyn isa::TargetIsa>, + inst: Inst, + indent: usize, + ) -> fmt::Result { + PlainWriter.write_instruction(w, func, aliases, isa, inst, indent)?; + if let Some(comment) = self.entity_comments.get(&inst.into()) { + writeln!(w, "; {}", comment.replace('\n', "\n; "))?; + } + Ok(()) + } +} + +#[cfg(debug_assertions)] +impl<M: Module> FunctionCx<'_, '_, M> { + pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) { + self.clif_comments.add_global_comment(comment); + } + + pub(crate) fn add_comment<S: Into<String> + AsRef<str>, E: Into<AnyEntity>>( + &mut self, + entity: E, + comment: S, + ) { + self.clif_comments.add_comment(entity, comment); + } +} + +pub(crate) fn write_clif_file<'tcx>( + tcx: TyCtxt<'tcx>, + postfix: &str, + isa: Option<&dyn cranelift_codegen::isa::TargetIsa>, + instance: Instance<'tcx>, + context: &cranelift_codegen::Context, + mut clif_comments: &CommentWriter, +) { + use std::io::Write; + + if !cfg!(debug_assertions) + && !tcx + .sess + .opts + .output_types + .contains_key(&OutputType::LlvmAssembly) + { + return; + } + + let value_ranges = isa.map(|isa| { + context + .build_value_labels_ranges(isa) + .expect("value location ranges") + }); + + let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif"); + + match std::fs::create_dir(&clif_output_dir) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {} + res @ Err(_) => res.unwrap(), + } + + let clif_file_name = clif_output_dir.join(format!( + "{}.{}.clif", + tcx.symbol_name(instance).name, + postfix + )); + + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { + isa: Some(&*crate::build_isa( + tcx.sess, true, /* PIC doesn't matter here */ + )), + value_ranges: value_ranges.as_ref(), + }, + ) + .unwrap(); + + let res: std::io::Result<()> = try { + let mut file = std::fs::File::create(clif_file_name)?; + let target_triple = crate::target_triple(tcx.sess); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", target_triple)?; + writeln!(file, "")?; + file.write_all(clif.as_bytes())?; + }; + if let Err(err) = res { + tcx.sess.warn(&format!("err writing clif file: {}", err)); + } +} + +impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{:?}", self.instance.substs)?; + writeln!(f, "{:?}", self.local_map)?; + + let mut clif = String::new(); + ::cranelift_codegen::write::decorate_function( + &mut &self.clif_comments, + &mut clif, + &self.bcx.func, + &DisplayFunctionAnnotations::default(), + ) + .unwrap(); + writeln!(f, "\n{}", clif) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs new file mode 100644 index 00000000000..463afaf7cc5 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs @@ -0,0 +1,125 @@ +//! Locating various executables part of a C toolchain. + +use std::path::PathBuf; + +use rustc_middle::bug; +use rustc_session::Session; +use rustc_target::spec::LinkerFlavor; + +/// Tries to infer the path of a binary for the target toolchain from the linker name. +pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { + let (mut linker, _linker_flavor) = linker_and_flavor(sess); + let linker_file_name = linker + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker")); + + if linker_file_name == "ld.lld" { + if tool != "ld" { + linker.set_file_name(tool) + } + } else { + let tool_file_name = linker_file_name + .replace("ld", tool) + .replace("gcc", tool) + .replace("clang", tool) + .replace("cc", tool); + + linker.set_file_name(tool_file_name) + } + + linker +} + +// Adapted from https://github.com/rust-lang/rust/blob/5db778affee7c6600c8e7a177c48282dab3f6292/src/librustc_codegen_ssa/back/link.rs#L848-L931 +fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { + fn infer_from( + sess: &Session, + linker: Option<PathBuf>, + flavor: Option<LinkerFlavor>, + ) -> Option<(PathBuf, LinkerFlavor)> { + match (linker, flavor) { + (Some(linker), Some(flavor)) => Some((linker, flavor)), + // only the linker flavor is known; use the default linker for the selected flavor + (None, Some(flavor)) => Some(( + PathBuf::from(match flavor { + LinkerFlavor::Em => { + if cfg!(windows) { + "emcc.bat" + } else { + "emcc" + } + } + LinkerFlavor::Gcc => { + if cfg!(any(target_os = "solaris", target_os = "illumos")) { + // On historical Solaris systems, "cc" may have + // been Sun Studio, which is not flag-compatible + // with "gcc". This history casts a long shadow, + // and many modern illumos distributions today + // ship GCC as "gcc" without also making it + // available as "cc". + "gcc" + } else { + "cc" + } + } + LinkerFlavor::Ld => "ld", + LinkerFlavor::Msvc => "link.exe", + LinkerFlavor::Lld(_) => "lld", + LinkerFlavor::PtxLinker => "rust-ptx-linker", + }), + flavor, + )), + (Some(linker), None) => { + let stem = linker + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or_else(|| { + sess.fatal("couldn't extract file stem from specified linker") + }); + + let flavor = if stem == "emcc" { + LinkerFlavor::Em + } else if stem == "gcc" + || stem.ends_with("-gcc") + || stem == "clang" + || stem.ends_with("-clang") + { + LinkerFlavor::Gcc + } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { + LinkerFlavor::Ld + } else if stem == "link" || stem == "lld-link" { + LinkerFlavor::Msvc + } else if stem == "lld" || stem == "rust-lld" { + LinkerFlavor::Lld(sess.target.options.lld_flavor) + } else { + // fall back to the value in the target spec + sess.target.linker_flavor + }; + + Some((linker, flavor)) + } + (None, None) => None, + } + } + + // linker and linker flavor specified via command line have precedence over what the target + // specification specifies + if let Some(ret) = infer_from( + sess, + sess.opts.cg.linker.clone(), + sess.opts.cg.linker_flavor, + ) { + return ret; + } + + if let Some(ret) = infer_from( + sess, + sess.target.options.linker.clone().map(PathBuf::from), + Some(sess.target.linker_flavor), + ) { + return ret; + } + + bug!("Not enough information provided to determine how to invoke the linker"); +} diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs new file mode 100644 index 00000000000..37dca77bdbd --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/trap.rs @@ -0,0 +1,70 @@ +//! Helpers used to print a message and abort in case of certain panics and some detected UB. + +use crate::prelude::*; + +fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) { + let puts = fx + .cx + .module + .declare_function( + "puts", + Linkage::Import, + &Signature { + call_conv: CallConv::triple_default(fx.triple()), + params: vec![AbiParam::new(pointer_ty(fx.tcx))], + returns: vec![AbiParam::new(types::I32)], + }, + ) + .unwrap(); + let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + #[cfg(debug_assertions)] + { + fx.add_comment(puts, "puts"); + } + + let symbol_name = fx.tcx.symbol_name(fx.instance); + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg); + let msg_ptr = fx.anonymous_str("trap", &real_msg); + fx.bcx.ins().call(puts, &[msg_ptr]); +} + +/// Trap code: user1 +pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) { + codegen_print(fx, msg.as_ref()); + fx.bcx.ins().trap(TrapCode::User(1)); +} + +/// Use this for example when a function call should never return. This will fill the current block, +/// so you can **not** add instructions to it afterwards. +/// +/// Trap code: user65535 +pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) { + codegen_print(fx, msg.as_ref()); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); +} + +/// Like `trap_unreachable` but returns a fake value of the specified type. +/// +/// Trap code: user65535 +pub(crate) fn trap_unreachable_ret_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + dest_layout: TyAndLayout<'tcx>, + msg: impl AsRef<str>, +) -> CValue<'tcx> { + codegen_print(fx, msg.as_ref()); + let true_ = fx.bcx.ins().iconst(types::I32, 1); + fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached); + CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout) +} + +/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen. +/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions +/// to it afterwards. +/// +/// Trap code: user65535 +pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) { + codegen_print(fx, msg.as_ref()); + let true_ = fx.bcx.ins().iconst(types::I32, 1); + fx.bcx.ins().trapnz(true_, TrapCode::User(!0)); +} + diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs new file mode 100644 index 00000000000..c77ff5d56ba --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -0,0 +1,238 @@ +//! Codegen of the [`PointerCast::Unsize`] operation. +//! +//! [`PointerCast::Unsize`]: `rustc_middle::ty::adjustment::PointerCast::Unsize` + +use crate::prelude::*; + +// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/base.rs#L159-L307 + +/// Retrieve the information we are losing (making dynamic) in an unsizing +/// adjustment. +/// +/// The `old_info` argument is a bit funny. It is intended for use +/// in an upcast, where the new vtable for an object will be derived +/// from the old one. +pub(crate) fn unsized_info<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + source: Ty<'tcx>, + target: Ty<'tcx>, + old_info: Option<Value>, +) -> Value { + let (source, target) = + fx.tcx + .struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all()); + match (&source.kind(), &target.kind()) { + (&ty::Array(_, len), &ty::Slice(_)) => fx.bcx.ins().iconst( + fx.pointer_type, + len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64, + ), + (&ty::Dynamic(..), &ty::Dynamic(..)) => { + // For now, upcasts are limited to changes in marker + // traits, and hence never actually require an actual + // change to the vtable. + old_info.expect("unsized_info: missing old info for trait upcast") + } + (_, &ty::Dynamic(ref data, ..)) => { + crate::vtable::get_vtable(fx, fx.layout_of(source), data.principal()) + } + _ => bug!( + "unsized_info: invalid unsizing {:?} -> {:?}", + source, + target + ), + } +} + +/// Coerce `src` to `dst_ty`. `src_ty` must be a thin pointer. +fn unsize_thin_ptr<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + src: Value, + src_layout: TyAndLayout<'tcx>, + dst_layout: TyAndLayout<'tcx>, +) -> (Value, Value) { + match (&src_layout.ty.kind(), &dst_layout.ty.kind()) { + (&ty::Ref(_, a, _), &ty::Ref(_, b, _)) + | (&ty::Ref(_, a, _), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) + | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => { + assert!(!fx.layout_of(a).is_unsized()); + (src, unsized_info(fx, a, b, None)) + } + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if def_a.is_box() && def_b.is_box() => { + let (a, b) = (src_layout.ty.boxed_ty(), dst_layout.ty.boxed_ty()); + assert!(!fx.layout_of(a).is_unsized()); + (src, unsized_info(fx, a, b, None)) + } + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + + let mut result = None; + for i in 0..src_layout.fields.count() { + let src_f = src_layout.field(fx, i); + assert_eq!(src_layout.fields.offset(i).bytes(), 0); + assert_eq!(dst_layout.fields.offset(i).bytes(), 0); + if src_f.is_zst() { + continue; + } + assert_eq!(src_layout.size, src_f.size); + + let dst_f = dst_layout.field(fx, i); + assert_ne!(src_f.ty, dst_f.ty); + assert_eq!(result, None); + result = Some(unsize_thin_ptr(fx, src, src_f, dst_f)); + } + result.unwrap() + } + _ => bug!("unsize_thin_ptr: called on bad types"), + } +} + +/// Coerce `src`, which is a reference to a value of type `src_ty`, +/// to a value of type `dst_ty` and store the result in `dst` +pub(crate) fn coerce_unsized_into<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + src: CValue<'tcx>, + dst: CPlace<'tcx>, +) { + let src_ty = src.layout().ty; + let dst_ty = dst.layout().ty; + let mut coerce_ptr = || { + let (base, info) = if fx + .layout_of(src.layout().ty.builtin_deref(true).unwrap().ty) + .is_unsized() + { + // fat-ptr to fat-ptr unsize preserves the vtable + // i.e., &'a fmt::Debug+Send => &'a fmt::Debug + src.load_scalar_pair(fx) + } else { + let base = src.load_scalar(fx); + unsize_thin_ptr(fx, base, src.layout(), dst.layout()) + }; + dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout())); + }; + match (&src_ty.kind(), &dst_ty.kind()) { + (&ty::Ref(..), &ty::Ref(..)) + | (&ty::Ref(..), &ty::RawPtr(..)) + | (&ty::RawPtr(..), &ty::RawPtr(..)) => coerce_ptr(), + (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { + assert_eq!(def_a, def_b); + + for i in 0..def_a.variants[VariantIdx::new(0)].fields.len() { + let src_f = src.value_field(fx, mir::Field::new(i)); + let dst_f = dst.place_field(fx, mir::Field::new(i)); + + if dst_f.layout().is_zst() { + continue; + } + + if src_f.layout().ty == dst_f.layout().ty { + dst_f.write_cvalue(fx, src_f); + } else { + coerce_unsized_into(fx, src_f, dst_f); + } + } + } + _ => bug!( + "coerce_unsized_into: invalid coercion {:?} -> {:?}", + src_ty, + dst_ty + ), + } +} + +// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs + +pub(crate) fn size_and_align_of_dst<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + info: Value, +) -> (Value, Value) { + if !layout.is_unsized() { + let size = fx + .bcx + .ins() + .iconst(fx.pointer_type, layout.size.bytes() as i64); + let align = fx + .bcx + .ins() + .iconst(fx.pointer_type, layout.align.abi.bytes() as i64); + return (size, align); + } + match layout.ty.kind() { + ty::Dynamic(..) => { + // load size/align from vtable + ( + crate::vtable::size_of_obj(fx, info), + crate::vtable::min_align_of_obj(fx, info), + ) + } + ty::Slice(_) | ty::Str => { + let unit = layout.field(fx, 0); + // The info in this case is the length of the str, so the size is that + // times the unit size. + ( + fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64), + fx.bcx + .ins() + .iconst(fx.pointer_type, unit.align.abi.bytes() as i64), + ) + } + _ => { + // First get the size of all statically known fields. + // Don't use size_of because it also rounds up to alignment, which we + // want to avoid, as the unsized field's alignment could be smaller. + assert!(!layout.ty.is_simd()); + + let i = layout.fields.count() - 1; + let sized_size = layout.fields.offset(i).bytes(); + let sized_align = layout.align.abi.bytes(); + let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64); + + // Recurse to get the size of the dynamically sized field (must be + // the last field). + let field_layout = layout.field(fx, i); + let (unsized_size, mut unsized_align) = size_and_align_of_dst(fx, field_layout, info); + + // FIXME (#26403, #27023): We should be adding padding + // to `sized_size` (to accommodate the `unsized_align` + // required of the unsized field that follows) before + // summing it with `sized_size`. (Note that since #26403 + // is unfixed, we do not yet add the necessary padding + // here. But this is where the add would go.) + + // Return the sum of sizes and max of aligns. + let size = fx.bcx.ins().iadd_imm(unsized_size, sized_size as i64); + + // Packed types ignore the alignment of their fields. + if let ty::Adt(def, _) = layout.ty.kind() { + if def.repr.packed() { + unsized_align = sized_align; + } + } + + // Choose max of two known alignments (combined value must + // be aligned according to more restrictive of the two). + let cmp = fx + .bcx + .ins() + .icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align); + let align = fx.bcx.ins().select(cmp, sized_align, unsized_align); + + // Issue #27023: must add any necessary padding to `size` + // (to make it a multiple of `align`) before returning it. + // + // Namely, the returned size should be, in C notation: + // + // `size + ((size & (align-1)) ? align : 0)` + // + // emulated via the semi-standard fast bit trick: + // + // `(size + (align-1)) & -align` + let addend = fx.bcx.ins().iadd_imm(align, -1); + let add = fx.bcx.ins().iadd(size, addend); + let neg = fx.bcx.ins().ineg(align); + let size = fx.bcx.ins().band(add, neg); + + (size, align) + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs new file mode 100644 index 00000000000..5d513cb3ea0 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -0,0 +1,777 @@ +//! Definition of [`CValue`] and [`CPlace`] + +use crate::prelude::*; + +use cranelift_codegen::entity::EntityRef; +use cranelift_codegen::ir::immediates::Offset32; + +fn codegen_field<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + base: Pointer, + extra: Option<Value>, + layout: TyAndLayout<'tcx>, + field: mir::Field, +) -> (Pointer, TyAndLayout<'tcx>) { + let field_offset = layout.fields.offset(field.index()); + let field_layout = layout.field(&*fx, field.index()); + + let simple = |fx: &mut FunctionCx<'_, '_, _>| { + ( + base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), + field_layout, + ) + }; + + if let Some(extra) = extra { + if !field_layout.is_unsized() { + return simple(fx); + } + match field_layout.ty.kind() { + ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(fx), + ty::Adt(def, _) if def.repr.packed() => { + assert_eq!(layout.align.abi.bytes(), 1); + return simple(fx); + } + _ => { + // We have to align the offset for DST's + let unaligned_offset = field_offset.bytes(); + let (_, unsized_align) = + crate::unsize::size_and_align_of_dst(fx, field_layout, extra); + + let one = fx.bcx.ins().iconst(pointer_ty(fx.tcx), 1); + let align_sub_1 = fx.bcx.ins().isub(unsized_align, one); + let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64); + let zero = fx.bcx.ins().iconst(pointer_ty(fx.tcx), 0); + let and_rhs = fx.bcx.ins().isub(zero, unsized_align); + let offset = fx.bcx.ins().band(and_lhs, and_rhs); + + (base.offset_value(fx, offset), field_layout) + } + } + } else { + simple(fx) + } +} + +fn scalar_pair_calculate_b_offset( + tcx: TyCtxt<'_>, + a_scalar: &Scalar, + b_scalar: &Scalar, +) -> Offset32 { + let b_offset = a_scalar + .value + .size(&tcx) + .align_to(b_scalar.value.align(&tcx).abi); + Offset32::new(b_offset.bytes().try_into().unwrap()) +} + +/// A read-only value +#[derive(Debug, Copy, Clone)] +pub(crate) struct CValue<'tcx>(CValueInner, TyAndLayout<'tcx>); + +#[derive(Debug, Copy, Clone)] +enum CValueInner { + ByRef(Pointer, Option<Value>), + ByVal(Value), + ByValPair(Value, Value), +} + +impl<'tcx> CValue<'tcx> { + pub(crate) fn by_ref(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { + CValue(CValueInner::ByRef(ptr, None), layout) + } + + pub(crate) fn by_ref_unsized( + ptr: Pointer, + meta: Value, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + CValue(CValueInner::ByRef(ptr, Some(meta)), layout) + } + + pub(crate) fn by_val(value: Value, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { + CValue(CValueInner::ByVal(value), layout) + } + + pub(crate) fn by_val_pair( + value: Value, + extra: Value, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + CValue(CValueInner::ByValPair(value, extra), layout) + } + + pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { + self.1 + } + + // FIXME remove + pub(crate) fn force_stack( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + ) -> (Pointer, Option<Value>) { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, meta) => (ptr, meta), + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => { + let cplace = CPlace::new_stack_slot(fx, layout); + cplace.write_cvalue(fx, self); + (cplace.to_ptr(), None) + } + } + } + + pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> { + match self.0 { + CValueInner::ByRef(ptr, meta) => Some((ptr, meta)), + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None, + } + } + + /// Load a value with layout.abi of scalar + pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> Value { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, None) => { + let clif_ty = match layout.abi { + Abi::Scalar(ref scalar) => scalar_to_clif_type(fx.tcx, scalar.clone()), + Abi::Vector { ref element, count } => { + scalar_to_clif_type(fx.tcx, element.clone()) + .by(u16::try_from(count).unwrap()) + .unwrap() + } + _ => unreachable!("{:?}", layout.ty), + }; + let mut flags = MemFlags::new(); + flags.set_notrap(); + ptr.load(fx, clif_ty, flags) + } + CValueInner::ByVal(value) => value, + CValueInner::ByRef(_, Some(_)) => bug!("load_scalar for unsized value not allowed"), + CValueInner::ByValPair(_, _) => bug!("Please use load_scalar_pair for ByValPair"), + } + } + + /// Load a value pair with layout.abi of scalar pair + pub(crate) fn load_scalar_pair( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + ) -> (Value, Value) { + let layout = self.1; + match self.0 { + CValueInner::ByRef(ptr, None) => { + let (a_scalar, b_scalar) = match &layout.abi { + Abi::ScalarPair(a, b) => (a, b), + _ => unreachable!("load_scalar_pair({:?})", self), + }; + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar.clone()); + let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar.clone()); + let mut flags = MemFlags::new(); + flags.set_notrap(); + let val1 = ptr.load(fx, clif_ty1, flags); + let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); + (val1, val2) + } + CValueInner::ByRef(_, Some(_)) => { + bug!("load_scalar_pair for unsized value not allowed") + } + CValueInner::ByVal(_) => bug!("Please use load_scalar for ByVal"), + CValueInner::ByValPair(val1, val2) => (val1, val2), + } + } + + pub(crate) fn value_field( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + field: mir::Field, + ) -> CValue<'tcx> { + let layout = self.1; + match self.0 { + CValueInner::ByVal(val) => match layout.abi { + Abi::Vector { element: _, count } => { + let count = u8::try_from(count).expect("SIMD type with more than 255 lanes???"); + let field = u8::try_from(field.index()).unwrap(); + assert!(field < count); + let lane = fx.bcx.ins().extractlane(val, field); + let field_layout = layout.field(&*fx, usize::from(field)); + CValue::by_val(lane, field_layout) + } + _ => unreachable!("value_field for ByVal with abi {:?}", layout.abi), + }, + CValueInner::ByValPair(val1, val2) => match layout.abi { + Abi::ScalarPair(_, _) => { + let val = match field.as_u32() { + 0 => val1, + 1 => val2, + _ => bug!("field should be 0 or 1"), + }; + let field_layout = layout.field(&*fx, usize::from(field)); + CValue::by_val(val, field_layout) + } + _ => unreachable!("value_field for ByValPair with abi {:?}", layout.abi), + }, + CValueInner::ByRef(ptr, None) => { + let (field_ptr, field_layout) = codegen_field(fx, ptr, None, layout, field); + CValue::by_ref(field_ptr, field_layout) + } + CValueInner::ByRef(_, Some(_)) => todo!(), + } + } + + pub(crate) fn unsize_value( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + dest: CPlace<'tcx>, + ) { + crate::unsize::coerce_unsized_into(fx, self, dest); + } + + /// If `ty` is signed, `const_val` must already be sign extended. + pub(crate) fn const_val( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + const_val: u128, + ) -> CValue<'tcx> { + use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; + + let clif_ty = fx.clif_type(layout.ty).unwrap(); + + match layout.ty.kind() { + ty::Bool => { + assert!( + const_val == 0 || const_val == 1, + "Invalid bool 0x{:032X}", + const_val + ); + } + _ => {} + } + + let val = match layout.ty.kind() { + ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { + let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64); + let msb = fx + .bcx + .ins() + .iconst(types::I64, (const_val >> 64) as u64 as i64); + fx.bcx.ins().iconcat(lsb, msb) + } + ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) + | ty::RawPtr(..) => { + fx + .bcx + .ins() + .iconst(clif_ty, u64::try_from(const_val).expect("uint") as i64) + } + ty::Float(FloatTy::F32) => { + fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap())) + } + ty::Float(FloatTy::F64) => { + fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap())) + } + _ => panic!( + "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed", + layout.ty + ), + }; + + CValue::by_val(val, layout) + } + + pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self { + assert!(matches!( + self.layout().ty.kind(), + ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) + )); + assert!(matches!( + layout.ty.kind(), + ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) + )); + assert_eq!(self.layout().abi, layout.abi); + CValue(self.0, layout) + } +} + +/// A place where you can write a value to or read a value from +#[derive(Debug, Copy, Clone)] +pub(crate) struct CPlace<'tcx> { + inner: CPlaceInner, + layout: TyAndLayout<'tcx>, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) enum CPlaceInner { + Var(Local, Variable), + VarPair(Local, Variable, Variable), + VarLane(Local, Variable, u8), + Addr(Pointer, Option<Value>), +} + +impl<'tcx> CPlace<'tcx> { + pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { + self.layout + } + + pub(crate) fn inner(&self) -> &CPlaceInner { + &self.inner + } + + pub(crate) fn no_place(layout: TyAndLayout<'tcx>) -> CPlace<'tcx> { + CPlace { + inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), + layout, + } + } + + pub(crate) fn new_stack_slot( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + assert!(!layout.is_unsized()); + if layout.size.bytes() == 0 { + return CPlace::no_place(layout); + } + + let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + kind: StackSlotKind::ExplicitSlot, + size: layout.size.bytes() as u32, + offset: None, + }); + CPlace { + inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), + layout, + } + } + + pub(crate) fn new_var( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + local: Local, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + let var = Variable::with_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap()); + CPlace { + inner: CPlaceInner::Var(local, var), + layout, + } + } + + pub(crate) fn new_var_pair( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + local: Local, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + let var1 = Variable::with_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + let var2 = Variable::with_u32(fx.next_ssa_var); + fx.next_ssa_var += 1; + + let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap(); + fx.bcx.declare_var(var1, ty1); + fx.bcx.declare_var(var2, ty2); + CPlace { + inner: CPlaceInner::VarPair(local, var1, var2), + layout, + } + } + + pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> { + CPlace { + inner: CPlaceInner::Addr(ptr, None), + layout, + } + } + + pub(crate) fn for_ptr_with_extra( + ptr: Pointer, + extra: Value, + layout: TyAndLayout<'tcx>, + ) -> CPlace<'tcx> { + CPlace { + inner: CPlaceInner::Addr(ptr, Some(extra)), + layout, + } + } + + pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CValue<'tcx> { + let layout = self.layout(); + match self.inner { + CPlaceInner::Var(_local, var) => { + let val = fx.bcx.use_var(var); + fx.bcx + .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); + CValue::by_val(val, layout) + } + CPlaceInner::VarPair(_local, var1, var2) => { + let val1 = fx.bcx.use_var(var1); + fx.bcx + .set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index())); + let val2 = fx.bcx.use_var(var2); + fx.bcx + .set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index())); + CValue::by_val_pair(val1, val2, layout) + } + CPlaceInner::VarLane(_local, var, lane) => { + let val = fx.bcx.use_var(var); + fx.bcx + .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); + let val = fx.bcx.ins().extractlane(val, lane); + CValue::by_val(val, layout) + } + CPlaceInner::Addr(ptr, extra) => { + if let Some(extra) = extra { + CValue::by_ref_unsized(ptr, extra, layout) + } else { + CValue::by_ref(ptr, layout) + } + } + } + } + + pub(crate) fn to_ptr(self) -> Pointer { + match self.to_ptr_maybe_unsized() { + (ptr, None) => ptr, + (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), + } + } + + pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) { + match self.inner { + CPlaceInner::Addr(ptr, extra) => (ptr, extra), + CPlaceInner::Var(_, _) + | CPlaceInner::VarPair(_, _, _) + | CPlaceInner::VarLane(_, _, _) => bug!("Expected CPlace::Addr, found {:?}", self), + } + } + + pub(crate) fn write_cvalue( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + from: CValue<'tcx>, + ) { + fn assert_assignable<'tcx>( + fx: &FunctionCx<'_, 'tcx, impl Module>, + from_ty: Ty<'tcx>, + to_ty: Ty<'tcx>, + ) { + match (&from_ty.kind(), &to_ty.kind()) { + (ty::Ref(_, a, _), ty::Ref(_, b, _)) + | ( + ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), + ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), + ) => { + assert_assignable(fx, a, b); + } + (ty::FnPtr(_), ty::FnPtr(_)) => { + let from_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + &from_ty.fn_sig(fx.tcx), + ); + let to_sig = fx.tcx.normalize_erasing_late_bound_regions( + ParamEnv::reveal_all(), + &to_ty.fn_sig(fx.tcx), + ); + assert_eq!( + from_sig, to_sig, + "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", + from_sig, to_sig, fx, + ); + // fn(&T) -> for<'l> fn(&'l T) is allowed + } + (ty::Dynamic(from_traits, _), ty::Dynamic(to_traits, _)) => { + let from_traits = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from_traits); + let to_traits = fx + .tcx + .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_traits); + assert_eq!( + from_traits, to_traits, + "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", + from_traits, to_traits, fx, + ); + // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed + } + _ => { + assert_eq!( + from_ty, + to_ty, + "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", + from_ty, + to_ty, + fx, + ); + } + } + } + + assert_assignable(fx, from.layout().ty, self.layout().ty); + + self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); + } + + pub(crate) fn write_cvalue_transmute( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + from: CValue<'tcx>, + ) { + self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute"); + } + + fn write_cvalue_maybe_transmute( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + from: CValue<'tcx>, + #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str, + ) { + fn transmute_value<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + var: Variable, + data: Value, + dst_ty: Type, + ) { + let src_ty = fx.bcx.func.dfg.value_type(data); + let data = match (src_ty, dst_ty) { + (_, _) if src_ty == dst_ty => data, + + // This is a `write_cvalue_transmute`. + (types::I32, types::F32) + | (types::F32, types::I32) + | (types::I64, types::F64) + | (types::F64, types::I64) => fx.bcx.ins().bitcast(dst_ty, data), + _ if src_ty.is_vector() && dst_ty.is_vector() => { + fx.bcx.ins().raw_bitcast(dst_ty, data) + } + _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty), + }; + fx.bcx + .set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index())); + fx.bcx.def_var(var, data); + } + + assert_eq!(self.layout().size, from.layout().size); + + #[cfg(debug_assertions)] + { + use cranelift_codegen::cursor::{Cursor, CursorPosition}; + let cur_block = match fx.bcx.cursor().position() { + CursorPosition::After(block) => block, + _ => unreachable!(), + }; + fx.add_comment( + fx.bcx.func.layout.last_inst(cur_block).unwrap(), + format!( + "{}: {:?}: {:?} <- {:?}: {:?}", + method, + self.inner(), + self.layout().ty, + from.0, + from.layout().ty + ), + ); + } + + let dst_layout = self.layout(); + let to_ptr = match self.inner { + CPlaceInner::Var(_local, var) => { + let data = CValue(from.0, dst_layout).load_scalar(fx); + let dst_ty = fx.clif_type(self.layout().ty).unwrap(); + transmute_value(fx, var, data, dst_ty); + return; + } + CPlaceInner::VarPair(_local, var1, var2) => { + let (data1, data2) = CValue(from.0, dst_layout).load_scalar_pair(fx); + let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); + transmute_value(fx, var1, data1, dst_ty1); + transmute_value(fx, var2, data2, dst_ty2); + return; + } + CPlaceInner::VarLane(_local, var, lane) => { + let data = from.load_scalar(fx); + + // First get the old vector + let vector = fx.bcx.use_var(var); + fx.bcx + .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); + + // Next insert the written lane into the vector + let vector = fx.bcx.ins().insertlane(vector, data, lane); + + // Finally write the new vector + fx.bcx + .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); + fx.bcx.def_var(var, vector); + + return; + } + CPlaceInner::Addr(ptr, None) => { + if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited { + return; + } + ptr + } + CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self), + }; + + let mut flags = MemFlags::new(); + flags.set_notrap(); + match from.layout().abi { + // FIXME make Abi::Vector work too + Abi::Scalar(_) => { + let val = from.load_scalar(fx); + to_ptr.store(fx, val, flags); + return; + } + Abi::ScalarPair(ref a_scalar, ref b_scalar) => { + let (value, extra) = from.load_scalar_pair(fx); + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + to_ptr.store(fx, value, flags); + to_ptr.offset(fx, b_offset).store(fx, extra, flags); + return; + } + _ => {} + } + + match from.0 { + CValueInner::ByVal(val) => { + to_ptr.store(fx, val, flags); + } + CValueInner::ByValPair(_, _) => { + bug!( + "Non ScalarPair abi {:?} for ByValPair CValue", + dst_layout.abi + ); + } + CValueInner::ByRef(from_ptr, None) => { + let from_addr = from_ptr.get_addr(fx); + let to_addr = to_ptr.get_addr(fx); + let src_layout = from.1; + let size = dst_layout.size.bytes(); + let src_align = src_layout.align.abi.bytes() as u8; + let dst_align = dst_layout.align.abi.bytes() as u8; + fx.bcx.emit_small_memory_copy( + fx.cx.module.target_config(), + to_addr, + from_addr, + size, + dst_align, + src_align, + true, + ); + } + CValueInner::ByRef(_, Some(_)) => todo!(), + } + } + + pub(crate) fn place_field( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + field: mir::Field, + ) -> CPlace<'tcx> { + let layout = self.layout(); + + match self.inner { + CPlaceInner::Var(local, var) => { + if let Abi::Vector { .. } = layout.abi { + return CPlace { + inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()), + layout: layout.field(fx, field.as_u32().try_into().unwrap()), + }; + } + } + CPlaceInner::VarPair(local, var1, var2) => { + let layout = layout.field(&*fx, field.index()); + + match field.as_u32() { + 0 => { + return CPlace { + inner: CPlaceInner::Var(local, var1), + layout, + } + } + 1 => { + return CPlace { + inner: CPlaceInner::Var(local, var2), + layout, + } + } + _ => unreachable!("field should be 0 or 1"), + } + } + _ => {} + } + + let (base, extra) = self.to_ptr_maybe_unsized(); + + let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); + if field_layout.is_unsized() { + CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) + } else { + CPlace::for_ptr(field_ptr, field_layout) + } + } + + pub(crate) fn place_index( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + index: Value, + ) -> CPlace<'tcx> { + let (elem_layout, ptr) = match self.layout().ty.kind() { + ty::Array(elem_ty, _) => (fx.layout_of(elem_ty), self.to_ptr()), + ty::Slice(elem_ty) => (fx.layout_of(elem_ty), self.to_ptr_maybe_unsized().0), + _ => bug!("place_index({:?})", self.layout().ty), + }; + + let offset = fx + .bcx + .ins() + .imul_imm(index, elem_layout.size.bytes() as i64); + + CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout) + } + + pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CPlace<'tcx> { + let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty); + if has_ptr_meta(fx.tcx, inner_layout.ty) { + let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); + CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) + } else { + CPlace::for_ptr( + Pointer::new(self.to_cvalue(fx).load_scalar(fx)), + inner_layout, + ) + } + } + + pub(crate) fn place_ref( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + ) -> CValue<'tcx> { + if has_ptr_meta(fx.tcx, self.layout().ty) { + let (ptr, extra) = self.to_ptr_maybe_unsized(); + CValue::by_val_pair( + ptr.get_addr(fx), + extra.expect("unsized type without metadata"), + layout, + ) + } else { + CValue::by_val(self.to_ptr().get_addr(fx), layout) + } + } + + pub(crate) fn downcast_variant( + self, + fx: &FunctionCx<'_, 'tcx, impl Module>, + variant: VariantIdx, + ) -> Self { + assert!(!self.layout().is_unsized()); + let layout = self.layout().for_variant(fx, variant); + CPlace { + inner: self.inner, + layout, + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs new file mode 100644 index 00000000000..bb3cf8b3f3a --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -0,0 +1,194 @@ +//! Codegen vtables and vtable accesses. +//! +//! See librustc_codegen_llvm/meth.rs for reference +// FIXME dedup this logic between miri, cg_llvm and cg_clif + +use crate::prelude::*; + +const DROP_FN_INDEX: usize = 0; +const SIZE_INDEX: usize = 1; +const ALIGN_INDEX: usize = 2; + +fn vtable_memflags() -> MemFlags { + let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap. + flags.set_readonly(); // A vtable is always read-only. + flags +} + +pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + pointer_ty(fx.tcx), + vtable_memflags(), + vtable, + (DROP_FN_INDEX * usize_size) as i32, + ) +} + +pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + pointer_ty(fx.tcx), + vtable_memflags(), + vtable, + (SIZE_INDEX * usize_size) as i32, + ) +} + +pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value { + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + fx.bcx.ins().load( + pointer_ty(fx.tcx), + vtable_memflags(), + vtable, + (ALIGN_INDEX * usize_size) as i32, + ) +} + +pub(crate) fn get_ptr_and_method_ref<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + arg: CValue<'tcx>, + idx: usize, +) -> (Value, Value) { + let (ptr, vtable) = if let Abi::ScalarPair(_, _) = arg.layout().abi { + arg.load_scalar_pair(fx) + } else { + let (ptr, vtable) = arg.try_to_ptr().unwrap(); + (ptr.get_addr(fx), vtable.unwrap()) + }; + + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes(); + let func_ref = fx.bcx.ins().load( + pointer_ty(fx.tcx), + vtable_memflags(), + vtable, + ((idx + 3) * usize_size as usize) as i32, + ); + (ptr, func_ref) +} + +pub(crate) fn get_vtable<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, +) -> Value { + let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) { + *data_id + } else { + let data_id = build_vtable(fx, layout, trait_ref); + fx.cx.vtables.insert((layout.ty, trait_ref), data_id); + data_id + }; + + let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.bcx.ins().global_value(fx.pointer_type, local_data_id) +} + +fn build_vtable<'tcx>( + fx: &mut FunctionCx<'_, 'tcx, impl Module>, + layout: TyAndLayout<'tcx>, + trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>, +) -> DataId { + let tcx = fx.tcx; + let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; + + let drop_in_place_fn = import_function( + tcx, + &mut fx.cx.module, + Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), + ); + + let mut components: Vec<_> = vec![Some(drop_in_place_fn), None, None]; + + let methods_root; + let methods = if let Some(trait_ref) = trait_ref { + methods_root = tcx.vtable_methods(trait_ref.with_self_ty(tcx, layout.ty)); + methods_root.iter() + } else { + (&[]).iter() + }; + let methods = methods.cloned().map(|opt_mth| { + opt_mth.map_or(None, |(def_id, substs)| { + Some(import_function( + tcx, + &mut fx.cx.module, + Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) + .unwrap() + .polymorphize(fx.tcx), + )) + }) + }); + components.extend(methods); + + let mut data_ctx = DataContext::new(); + let mut data = ::std::iter::repeat(0u8) + .take(components.len() * usize_size) + .collect::<Vec<u8>>() + .into_boxed_slice(); + + write_usize(fx.tcx, &mut data, SIZE_INDEX, layout.size.bytes()); + write_usize(fx.tcx, &mut data, ALIGN_INDEX, layout.align.abi.bytes()); + data_ctx.define(data); + + for (i, component) in components.into_iter().enumerate() { + if let Some(func_id) = component { + let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + data_ctx.write_function_addr((i * usize_size) as u32, func_ref); + } + } + + data_ctx.set_align( + fx.tcx + .data_layout + .pointer_align + .pref + .bytes() + .try_into() + .unwrap(), + ); + + let data_id = fx + .cx + .module + .declare_data( + &format!( + "__vtable.{}.for.{:?}.{}", + trait_ref + .as_ref() + .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into()) + .unwrap_or(std::borrow::Cow::Borrowed("???")), + layout.ty, + fx.cx.vtables.len(), + ), + Linkage::Local, + false, + false, + ) + .unwrap(); + + fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + + data_id +} + +fn write_usize(tcx: TyCtxt<'_>, buf: &mut [u8], idx: usize, num: u64) { + let pointer_size = tcx + .layout_of(ParamEnv::reveal_all().and(tcx.types.usize)) + .unwrap() + .size + .bytes() as usize; + let target = &mut buf[idx * pointer_size..(idx + 1) * pointer_size]; + + match tcx.data_layout.endian { + rustc_target::abi::Endian::Little => match pointer_size { + 4 => target.copy_from_slice(&(num as u32).to_le_bytes()), + 8 => target.copy_from_slice(&(num as u64).to_le_bytes()), + _ => todo!("pointer size {} is not yet supported", pointer_size), + }, + rustc_target::abi::Endian::Big => match pointer_size { + 4 => target.copy_from_slice(&(num as u32).to_be_bytes()), + 8 => target.copy_from_slice(&(num as u64).to_be_bytes()), + _ => todo!("pointer size {} is not yet supported", pointer_size), + }, + } +} diff --git a/compiler/rustc_codegen_cranelift/test.sh b/compiler/rustc_codegen_cranelift/test.sh new file mode 100755 index 00000000000..a1c4d9f2872 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/test.sh @@ -0,0 +1,119 @@ +#!/bin/bash +set -e + +# Build cg_clif +export RUSTFLAGS="-Zrun_dsymutil=no" +if [[ "$1" == "--release" ]]; then + export CHANNEL='release' + cargo build --release +else + export CHANNEL='debug' + cargo build --bin cg_clif +fi + +# Config +source scripts/config.sh +export CG_CLIF_INCR_CACHE_DISABLED=1 +RUSTC=$RUSTC" "$RUSTFLAGS" -L crate=target/out --out-dir target/out -Cdebuginfo=2" + +# Cleanup +rm -r target/out || true + +# Perform all tests +echo "[BUILD] mini_core" +$RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE + +echo "[BUILD] example" +$RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE + +if [[ "$JIT_SUPPORTED" = "1" ]]; then + echo "[JIT] mini_core_hello_world" + CG_CLIF_JIT_ARGS="abc bcd" $RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target $HOST_TRIPLE +else + echo "[JIT] mini_core_hello_world (skipped)" +fi + +echo "[AOT] mini_core_hello_world" +$RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd +# (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd + +echo "[AOT] arbitrary_self_types_pointers_and_wrappers" +$RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers + +echo "[BUILD] sysroot" +time ./build_sysroot/build_sysroot.sh --release + +echo "[AOT] alloc_example" +$RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/alloc_example + +if [[ "$JIT_SUPPORTED" = "1" ]]; then + echo "[JIT] std_example" + $RUSTC --jit example/std_example.rs --target $HOST_TRIPLE +else + echo "[JIT] std_example (skipped)" +fi + +echo "[AOT] dst_field_align" +# FIXME Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. +$RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/dst_field_align || (echo $?; false) + +echo "[AOT] std_example" +$RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/std_example arg + +echo "[AOT] subslice-patterns-const-eval" +$RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/subslice-patterns-const-eval + +echo "[AOT] track-caller-attribute" +$RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/track-caller-attribute + +echo "[AOT] mod_bench" +$RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE +$RUN_WRAPPER ./target/out/mod_bench + +pushd rand +rm -r ./target || true +../cargo.sh test --workspace +popd + +pushd simple-raytracer +if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then + echo "[BENCH COMPILE] ebobby/simple-raytracer" + hyperfine --runs ${RUN_RUNS:-10} --warmup 1 --prepare "cargo clean" \ + "RUSTC=rustc RUSTFLAGS='' cargo build" \ + "../cargo.sh build" + + echo "[BENCH RUN] ebobby/simple-raytracer" + cp ./target/debug/main ./raytracer_cg_clif + hyperfine --runs ${RUN_RUNS:-10} ./raytracer_cg_llvm ./raytracer_cg_clif +else + echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)" + echo "[COMPILE] ebobby/simple-raytracer" + ../cargo.sh build + echo "[BENCH RUN] ebobby/simple-raytracer (skipped)" +fi +popd + +pushd build_sysroot/sysroot_src/library/core/tests +echo "[TEST] libcore" +rm -r ./target || true +../../../../../cargo.sh test +popd + +pushd regex +echo "[TEST] rust-lang/regex example shootout-regex-dna" +../cargo.sh clean +# Make sure `[codegen mono items] start` doesn't poison the diff +../cargo.sh build --example shootout-regex-dna +cat examples/regexdna-input.txt | ../cargo.sh run --example shootout-regex-dna | grep -v "Spawned thread" > res.txt +diff -u res.txt examples/regexdna-output.txt + +echo "[TEST] rust-lang/regex tests" +../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options +popd diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 04792b334d5..f9373640dce 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -11,11 +11,11 @@ doctest = false [dependencies] bitflags = "1.0" libc = "0.2" -measureme = "0.7.1" +measureme = "9.0.0" snap = "1" tracing = "0.1" rustc_middle = { path = "../rustc_middle" } -rustc-demangle = "0.1" +rustc-demangle = "0.1.18" rustc_attr = { path = "../rustc_attr" } rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index d02bc41f4af..f7d82ff78fa 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -93,7 +93,7 @@ pub(crate) unsafe fn codegen( let args = [usize, usize]; // size, align let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); - let name = format!("__rust_alloc_error_handler"); + let name = "__rust_alloc_error_handler".to_string(); let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); // -> ! DIFlagNoReturn llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index b096664bc74..d856280158f 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -302,13 +302,11 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { } else if options.contains(InlineAsmOptions::READONLY) { llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result); } + } else if options.contains(InlineAsmOptions::NOMEM) { + llvm::Attribute::InaccessibleMemOnly + .apply_callsite(llvm::AttributePlace::Function, result); } else { - if options.contains(InlineAsmOptions::NOMEM) { - llvm::Attribute::InaccessibleMemOnly - .apply_callsite(llvm::AttributePlace::Function, result); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect - } + // LLVM doesn't have an attribute to represent ReadOnly + SideEffect } // Write results to outputs diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index ff312bade25..64fd1d09cc2 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -900,7 +900,7 @@ impl ThinLTOKeysMap { let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; - let mut split = line.split(" "); + let mut split = line.split(' '); let module = split.next().unwrap(); let key = split.next().unwrap(); assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 092d1cea295..f13c2d312df 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -128,7 +128,8 @@ pub fn target_machine_factory( let (opt_level, _) = to_llvm_opt_settings(optlvl); let use_softfp = sess.opts.cg.soft_float; - let ffunction_sections = sess.target.options.function_sections; + let ffunction_sections = + sess.opts.debugging_opts.function_sections.unwrap_or(sess.target.options.function_sections); let fdata_sections = ffunction_sections; let code_model = to_llvm_code_model(sess.code_model()); @@ -155,7 +156,11 @@ pub fn target_machine_factory( let emit_stack_size_section = sess.opts.debugging_opts.emit_stack_sizes; let asm_comments = sess.asm_comments(); - let relax_elf_relocations = sess.target.options.relax_elf_relocations; + let relax_elf_relocations = sess + .opts + .debugging_opts + .relax_elf_relocations + .unwrap_or(sess.target.options.relax_elf_relocations); let use_init_array = !sess .opts @@ -936,8 +941,8 @@ unsafe fn embed_bitcode( llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } else { let asm = " - .section .llvmbc,\"a\" - .section .llvmcmd,\"a\" + .section .llvmbc,\"e\" + .section .llvmcmd,\"e\" "; llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 491191d058c..f122fa14e70 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -56,6 +56,7 @@ impl BackendTypes for Builder<'_, 'll, 'tcx> { type Funclet = <CodegenCx<'ll, 'tcx> as BackendTypes>::Funclet; type DIScope = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIScope; + type DILocation = <CodegenCx<'ll, 'tcx> as BackendTypes>::DILocation; type DIVariable = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIVariable; } @@ -731,10 +732,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let src_ty = self.cx.val_ty(val); let float_width = self.cx.float_width(src_ty); let int_width = self.cx.int_width(dest_ty); - match (int_width, float_width) { - (32, 32) | (32, 64) | (64, 32) | (64, 64) => true, - _ => false, - } + matches!((int_width, float_width), (32, 32) | (32, 64) | (64, 32) | (64, 64)) } fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 0992410a728..7633dd0600d 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -80,6 +80,7 @@ impl Funclet<'ll> { impl BackendTypes for CodegenCx<'ll, 'tcx> { type Value = &'ll Value; + // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`. type Function = &'ll Value; type BasicBlock = &'ll BasicBlock; @@ -87,6 +88,7 @@ impl BackendTypes for CodegenCx<'ll, 'tcx> { type Funclet = Funclet<'ll>; type DIScope = &'ll llvm::debuginfo::DIScope; + type DILocation = &'ll llvm::debuginfo::DILocation; type DIVariable = &'ll llvm::debuginfo::DIVariable; } diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index b57a23328b6..1672601d36f 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -397,10 +397,8 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { // As an optimization, all shared statics which do not have interior // mutability are placed into read-only memory. - if !is_mutable { - if self.type_is_freeze(ty) { - llvm::LLVMSetGlobalConstant(g, llvm::True); - } + if !is_mutable && self.type_is_freeze(ty) { + llvm::LLVMSetGlobalConstant(g, llvm::True); } debuginfo::create_global_var_metadata(&self, def_id, g); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 150cedde7e8..253e02bd082 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -122,10 +122,10 @@ pub unsafe fn create_module( if llvm_util::get_major_version() < 9 { target_data_layout = strip_function_ptr_alignment(target_data_layout); } - if llvm_util::get_major_version() < 10 { - if sess.target.arch == "x86" || sess.target.arch == "x86_64" { - target_data_layout = strip_x86_address_spaces(target_data_layout); - } + if llvm_util::get_major_version() < 10 + && (sess.target.arch == "x86" || sess.target.arch == "x86_64") + { + target_data_layout = strip_x86_address_spaces(target_data_layout); } // Ensure the data-layout values hardcoded remain the defaults. @@ -324,8 +324,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } #[inline] - pub fn coverage_context(&'a self) -> &'a coverageinfo::CrateCoverageContext<'tcx> { - self.coverage_cx.as_ref().unwrap() + pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'tcx>> { + self.coverage_cx.as_ref() } } @@ -864,7 +864,7 @@ impl<'b, 'tcx> CodegenCx<'b, 'tcx> { // user defined names let mut name = String::with_capacity(prefix.len() + 6); name.push_str(prefix); - name.push_str("."); + name.push('.'); base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); name } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 0098555a373..c1163a871cf 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -26,7 +26,10 @@ use tracing::debug; /// undocumented details in Clang's implementation (that may or may not be important) were also /// replicated for Rust's Coverage Map. pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { - let function_coverage_map = cx.coverage_context().take_function_coverage_map(); + let function_coverage_map = match cx.coverage_context() { + Some(ctx) => ctx.take_function_coverage_map(), + None => return, + }; if function_coverage_map.is_empty() { // This module has no functions with coverage instrumentation return; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 2bd37bf9c4f..7fdbe1a5512 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -64,17 +64,22 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { function_source_hash: u64, id: CounterValueReference, region: CodeRegion, - ) { - debug!( - "adding counter to coverage_regions: instance={:?}, function_source_hash={}, id={:?}, \ - at {:?}", - instance, function_source_hash, id, region, - ); - let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); - coverage_regions - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter(function_source_hash, id, region); + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter to coverage_regions: instance={:?}, function_source_hash={}, id={:?}, \ + at {:?}", + instance, function_source_hash, id, region, + ); + let mut coverage_regions = coverage_context.function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter(function_source_hash, id, region); + true + } else { + false + } } fn add_counter_expression_region( @@ -85,29 +90,39 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { op: Op, rhs: ExpressionOperandId, region: CodeRegion, - ) { - debug!( - "adding counter expression to coverage_regions: instance={:?}, id={:?}, {:?} {:?} {:?}, \ - at {:?}", - instance, id, lhs, op, rhs, region, - ); - let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); - coverage_regions - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_counter_expression(id, lhs, op, rhs, region); + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter expression to coverage_regions: instance={:?}, id={:?}, {:?} {:?} {:?}, \ + at {:?}", + instance, id, lhs, op, rhs, region, + ); + let mut coverage_regions = coverage_context.function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter_expression(id, lhs, op, rhs, region); + true + } else { + false + } } - fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion) { - debug!( - "adding unreachable code to coverage_regions: instance={:?}, at {:?}", - instance, region, - ); - let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); - coverage_regions - .entry(instance) - .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) - .add_unreachable_region(region); + fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding unreachable code to coverage_regions: instance={:?}, at {:?}", + instance, region, + ); + let mut coverage_regions = coverage_context.function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_unreachable_region(region); + true + } else { + false + } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 7f47b61de3f..6737872f203 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -3,21 +3,26 @@ use super::utils::DIB; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; use rustc_codegen_ssa::traits::*; +use crate::abi::FnAbi; use crate::common::CodegenCx; use crate::llvm; -use crate::llvm::debuginfo::{DIScope, DISubprogram}; +use crate::llvm::debuginfo::{DILocation, DIScope}; use rustc_middle::mir::{Body, SourceScope}; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; use rustc_index::bit_set::BitSet; use rustc_index::vec::Idx; /// Produces DIScope DIEs for each MIR Scope which has variables defined in it. +// FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. pub fn compute_mir_scopes( - cx: &CodegenCx<'ll, '_>, - mir: &Body<'_>, - fn_metadata: &'ll DISubprogram, - debug_context: &mut FunctionDebugContext<&'ll DIScope>, + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + fn_dbg_scope: &'ll DIScope, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, ) { // Find all the scopes with variables defined in them. let mut has_variables = BitSet::new_empty(mir.source_scopes.len()); @@ -37,58 +42,82 @@ pub fn compute_mir_scopes( // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, &mir, fn_metadata, &has_variables, debug_context, scope); + make_mir_scope(cx, instance, &mir, fn_dbg_scope, &has_variables, debug_context, scope); } } fn make_mir_scope( - cx: &CodegenCx<'ll, '_>, - mir: &Body<'_>, - fn_metadata: &'ll DISubprogram, + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + fn_dbg_scope: &'ll DIScope, has_variables: &BitSet<SourceScope>, - debug_context: &mut FunctionDebugContext<&'ll DISubprogram>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, scope: SourceScope, ) { - if debug_context.scopes[scope].is_valid() { + if debug_context.scopes[scope].dbg_scope.is_some() { return; } let scope_data = &mir.source_scopes[scope]; let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, mir, fn_metadata, has_variables, debug_context, parent); + make_mir_scope(cx, instance, mir, fn_dbg_scope, has_variables, debug_context, parent); debug_context.scopes[parent] } else { // The root is the function itself. let loc = cx.lookup_debug_loc(mir.span.lo()); debug_context.scopes[scope] = DebugScope { - scope_metadata: Some(fn_metadata), + dbg_scope: Some(fn_dbg_scope), + inlined_at: None, file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, }; return; }; - if !has_variables.contains(scope) { - // Do not create a DIScope if there are no variables - // defined in this MIR Scope, to avoid debuginfo bloat. + if !has_variables.contains(scope) && scope_data.inlined.is_none() { + // Do not create a DIScope if there are no variables defined in this + // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. debug_context.scopes[scope] = parent_scope; return; } let loc = cx.lookup_debug_loc(scope_data.span.lo()); - let file_metadata = file_metadata(cx, &loc.file, debug_context.defining_crate); + let file_metadata = file_metadata(cx, &loc.file); - let scope_metadata = unsafe { - Some(llvm::LLVMRustDIBuilderCreateLexicalBlock( - DIB(cx), - parent_scope.scope_metadata.unwrap(), - file_metadata, - loc.line.unwrap_or(UNKNOWN_LINE_NUMBER), - loc.col.unwrap_or(UNKNOWN_COLUMN_NUMBER), - )) + let dbg_scope = match scope_data.inlined { + Some((callee, _)) => { + // FIXME(eddyb) this would be `self.monomorphize(&callee)` + // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. + let callee = cx.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + &callee, + ); + let callee_fn_abi = FnAbi::of_instance(cx, callee, &[]); + cx.dbg_scope_fn(callee, &callee_fn_abi, None) + } + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope.unwrap(), + file_metadata, + loc.line.unwrap_or(UNKNOWN_LINE_NUMBER), + loc.col.unwrap_or(UNKNOWN_COLUMN_NUMBER), + ) + }, }; + + let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { + // FIXME(eddyb) this doesn't account for the macro-related + // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. + let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); + cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) + }); + debug_context.scopes[scope] = DebugScope { - scope_metadata, + dbg_scope: Some(dbg_scope), + inlined_at: inlined_at.or(parent_scope.inlined_at), file_start_pos: loc.file.start_pos, file_end_pos: loc.file.end_pos, }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs b/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs index b3a8fa29887..10dd5906529 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs @@ -28,7 +28,7 @@ //! utilizing a cache. The way to get a shared metadata node when needed is //! thus to just call the corresponding function in this module: //! -//! let file_metadata = file_metadata(crate_context, path); +//! let file_metadata = file_metadata(cx, file); //! //! The function will take care of probing the cache for an existing node for //! that exact file path. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5587e6ead1d..73c1f73ec7f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -26,7 +26,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_fs_util::path_to_c_string; use rustc_hir::def::CtorKind; -use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::ich::NodeIdHashingMode; use rustc_middle::mir::interpret::truncate; @@ -760,16 +760,12 @@ fn hex_encode(data: &[u8]) -> String { hex_string } -pub fn file_metadata( - cx: &CodegenCx<'ll, '_>, - source_file: &SourceFile, - defining_crate: CrateNum, -) -> &'ll DIFile { - debug!("file_metadata: file_name: {}, defining_crate: {}", source_file.name, defining_crate); +pub fn file_metadata(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { + debug!("file_metadata: file_name: {}", source_file.name); let hash = Some(&source_file.src_hash); let file_name = Some(source_file.name.to_string()); - let directory = if defining_crate == LOCAL_CRATE { + let directory = if source_file.is_real_file() && !source_file.is_imported() { Some(cx.sess().working_dir.0.to_string_lossy().to_string()) } else { // If the path comes from an upstream crate we assume it has been made @@ -1835,7 +1831,7 @@ impl<'tcx> VariantInfo<'_, 'tcx> { if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); return Some(SourceInfo { - file: file_metadata(cx, &loc.file, def_id.krate), + file: file_metadata(cx, &loc.file), line: loc.line.unwrap_or(UNKNOWN_LINE_NUMBER), }); } @@ -2474,7 +2470,7 @@ pub fn create_global_var_metadata(cx: &CodegenCx<'ll, '_>, def_id: DefId, global let (file_metadata, line_number) = if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); - (file_metadata(cx, &loc.file, LOCAL_CRATE), loc.line) + (file_metadata(cx, &loc.file), loc.line) } else { (unknown_file_metadata(cx), None) }; @@ -2576,9 +2572,8 @@ pub fn create_vtable_metadata(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, vtable: & pub fn extend_scope_to_file( cx: &CodegenCx<'ll, '_>, scope_metadata: &'ll DIScope, - file: &rustc_span::SourceFile, - defining_crate: CrateNum, + file: &SourceFile, ) -> &'ll DILexicalBlock { - let file_metadata = file_metadata(cx, &file, defining_crate); + let file_metadata = file_metadata(cx, file); unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 80e0e7bf2e0..6bb93857d71 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -3,7 +3,8 @@ mod doc; use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; -use self::metadata::{file_metadata, type_metadata, TypeMap, UNKNOWN_LINE_NUMBER}; +use self::metadata::{file_metadata, type_metadata, TypeMap}; +use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; use self::namespace::mangled_name_of_instance; use self::type_names::compute_debuginfo_type_name; use self::utils::{create_DIArray, is_node_local_to_unit, DIB}; @@ -13,7 +14,8 @@ use crate::builder::Builder; use crate::common::CodegenCx; use crate::llvm; use crate::llvm::debuginfo::{ - DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DISPFlags, DIScope, DIType, DIVariable, + DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, + DIVariable, }; use crate::value::Value; @@ -21,7 +23,8 @@ use rustc_codegen_ssa::debuginfo::type_names; use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; -use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE}; +use rustc_data_structures::sync::Lrc; +use rustc_hir::def_id::{DefId, DefIdMap, LOCAL_CRATE}; use rustc_index::vec::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::HasTyCtxt; @@ -29,7 +32,7 @@ use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable}; use rustc_session::config::{self, DebugInfo}; use rustc_span::symbol::Symbol; -use rustc_span::{self, BytePos, Span}; +use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, Span}; use rustc_target::abi::{LayoutOf, Primitive, Size}; use libc::c_uint; @@ -41,7 +44,6 @@ mod create_scope_map; pub mod gdb; pub mod metadata; mod namespace; -mod source_loc; mod utils; pub use self::create_scope_map::compute_mir_scopes; @@ -141,14 +143,11 @@ impl DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> { fn dbg_var_addr( &mut self, dbg_var: &'ll DIVariable, - scope_metadata: &'ll DIScope, + dbg_loc: &'ll DILocation, variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], - span: Span, ) { - let cx = self.cx(); - // Convert the direct and indirect offsets to address ops. // FIXME(eddyb) use `const`s instead of getting the values via FFI, // the values should match the ones in the DWARF standard anyway. @@ -168,14 +167,10 @@ impl DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> { } } - // FIXME(eddyb) maybe this information could be extracted from `dbg_var`, - // to avoid having to pass it down in both places? - // NB: `var` doesn't seem to know about the column, so that's a limitation. - let dbg_loc = cx.create_debug_loc(scope_metadata, span); unsafe { // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(cx), + DIB(self.cx()), variable_alloca, dbg_var, addr_ops.as_ptr(), @@ -186,15 +181,13 @@ impl DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> { } } - fn set_source_location(&mut self, scope: &'ll DIScope, span: Span) { - debug!("set_source_location: {}", self.sess().source_map().span_to_string(span)); - - let dbg_loc = self.cx().create_debug_loc(scope, span); - + fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { - llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc); + let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); + llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); } } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { gdb::insert_reference_to_gdb_debug_scripts_section_global(self) } @@ -223,30 +216,95 @@ impl DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> { } } +/// A source code location used to generate debug information. +// FIXME(eddyb) rename this to better indicate it's a duplicate of +// `rustc_span::Loc` rather than `DILocation`, perhaps by making +// `lookup_char_pos` return the right information instead. +pub struct DebugLoc { + /// Information about the original source file. + pub file: Lrc<SourceFile>, + /// The (1-based) line number. + pub line: Option<u32>, + /// The (1-based) column number. + pub col: Option<u32>, +} + +impl CodegenCx<'ll, '_> { + /// Looks up debug source information about a `BytePos`. + // FIXME(eddyb) rename this to better indicate it's a duplicate of + // `lookup_char_pos` rather than `dbg_loc`, perhaps by making + // `lookup_char_pos` return the right information instead. + pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { + let (file, line, col) = match self.sess().source_map().lookup_line(pos) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(pos); + + // Use 1-based indexing. + let line = (line + 1) as u32; + let col = (pos - line_pos).to_u32() + 1; + + (file, Some(line), Some(col)) + } + Err(file) => (file, None, None), + }; + + // For MSVC, omit the column number. + // Otherwise, emit it. This mimics clang behaviour. + // See discussion in https://github.com/rust-lang/rust/issues/42921 + if self.sess().target.options.is_like_msvc { + DebugLoc { file, line, col: None } + } else { + DebugLoc { file, line, col } + } + } +} + impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn create_function_debug_context( &self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: &'ll Value, - mir: &mir::Body<'_>, - ) -> Option<FunctionDebugContext<&'ll DIScope>> { + mir: &mir::Body<'tcx>, + ) -> Option<FunctionDebugContext<&'ll DIScope, &'ll DILocation>> { if self.sess().opts.debuginfo == DebugInfo::None { return None; } - let span = mir.span; + // Initialize fn debug context (including scopes). + // FIXME(eddyb) figure out a way to not need `Option` for `dbg_scope`. + let empty_scope = DebugScope { + dbg_scope: None, + inlined_at: None, + file_start_pos: BytePos(0), + file_end_pos: BytePos(0), + }; + let mut fn_debug_context = + FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; - // This can be the case for functions inlined from another crate - if span.is_dummy() { - // FIXME(simulacrum): Probably can't happen; remove. - return None; - } + // Fill in all the scopes, with the information from the MIR body. + compute_mir_scopes( + self, + instance, + mir, + self.dbg_scope_fn(instance, fn_abi, Some(llfn)), + &mut fn_debug_context, + ); + + Some(fn_debug_context) + } + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option<&'ll Value>, + ) -> &'ll DIScope { let def_id = instance.def_id(); let containing_scope = get_containing_scope(self, instance); + let span = self.tcx.def_span(def_id); let loc = self.lookup_debug_loc(span.lo()); - let file_metadata = file_metadata(self, &loc.file, def_id.krate); + let file_metadata = file_metadata(self, &loc.file); let function_type_metadata = unsafe { let fn_signature = get_function_signature(self, fn_abi); @@ -291,8 +349,8 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } - let fn_metadata = unsafe { - llvm::LLVMRustDIBuilderCreateFunction( + unsafe { + return llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, name.as_ptr().cast(), @@ -305,28 +363,11 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { scope_line.unwrap_or(UNKNOWN_LINE_NUMBER), flags, spflags, - llfn, + maybe_definition_llfn, template_parameters, None, - ) - }; - - // Initialize fn debug context (including scopes). - // FIXME(eddyb) figure out a way to not need `Option` for `scope_metadata`. - let null_scope = DebugScope { - scope_metadata: None, - file_start_pos: BytePos(0), - file_end_pos: BytePos(0), - }; - let mut fn_debug_context = FunctionDebugContext { - scopes: IndexVec::from_elem(null_scope, &mir.source_scopes), - defining_crate: def_id.krate, - }; - - // Fill in all the scopes, with the information from the MIR body. - compute_mir_scopes(self, mir, fn_metadata, &mut fn_debug_context); - - return Some(fn_debug_context); + ); + } fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, @@ -394,7 +435,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { name_to_append_suffix_to.push('<'); for (i, actual_type) in substs.types().enumerate() { if i != 0 { - name_to_append_suffix_to.push_str(","); + name_to_append_suffix_to.push(','); } let actual_type = @@ -501,6 +542,25 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } } + fn dbg_loc( + &self, + scope: &'ll DIScope, + inlined_at: Option<&'ll DILocation>, + span: Span, + ) -> &'ll DILocation { + let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + + unsafe { + llvm::LLVMRustDIBuilderCreateDebugLocation( + utils::debug_context(self).llcontext, + line.unwrap_or(UNKNOWN_LINE_NUMBER), + col.unwrap_or(UNKNOWN_COLUMN_NUMBER), + scope, + inlined_at, + ) + } + } + fn create_vtable_metadata(&self, ty: Ty<'tcx>, vtable: Self::Value) { metadata::create_vtable_metadata(self, ty, vtable) } @@ -509,9 +569,8 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { &self, scope_metadata: &'ll DIScope, file: &rustc_span::SourceFile, - defining_crate: CrateNum, ) -> &'ll DILexicalBlock { - metadata::extend_scope_to_file(&self, scope_metadata, file, defining_crate) + metadata::extend_scope_to_file(&self, scope_metadata, file) } fn debuginfo_finalize(&self) { @@ -522,7 +581,6 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). fn create_dbg_var( &self, - dbg_context: &FunctionDebugContext<&'ll DIScope>, variable_name: Symbol, variable_type: Ty<'tcx>, scope_metadata: &'ll DIScope, @@ -530,7 +588,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { span: Span, ) -> &'ll DIVariable { let loc = self.lookup_debug_loc(span.lo()); - let file_metadata = file_metadata(self, &loc.file, dbg_context.defining_crate); + let file_metadata = file_metadata(self, &loc.file); let type_metadata = type_metadata(self, variable_type, span); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/source_loc.rs b/compiler/rustc_codegen_llvm/src/debuginfo/source_loc.rs deleted file mode 100644 index 517246cd0b2..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/source_loc.rs +++ /dev/null @@ -1,61 +0,0 @@ -use super::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; -use super::utils::debug_context; - -use crate::common::CodegenCx; -use crate::llvm::debuginfo::DIScope; -use crate::llvm::{self, Value}; -use rustc_codegen_ssa::traits::*; - -use rustc_data_structures::sync::Lrc; -use rustc_span::{BytePos, Pos, SourceFile, SourceFileAndLine, Span}; - -/// A source code location used to generate debug information. -pub struct DebugLoc { - /// Information about the original source file. - pub file: Lrc<SourceFile>, - /// The (1-based) line number. - pub line: Option<u32>, - /// The (1-based) column number. - pub col: Option<u32>, -} - -impl CodegenCx<'ll, '_> { - /// Looks up debug source information about a `BytePos`. - pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { - let (file, line, col) = match self.sess().source_map().lookup_line(pos) { - Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.line_begin_pos(pos); - - // Use 1-based indexing. - let line = (line + 1) as u32; - let col = (pos - line_pos).to_u32() + 1; - - (file, Some(line), Some(col)) - } - Err(file) => (file, None, None), - }; - - // For MSVC, omit the column number. - // Otherwise, emit it. This mimics clang behaviour. - // See discussion in https://github.com/rust-lang/rust/issues/42921 - if self.sess().target.options.is_like_msvc { - DebugLoc { file, line, col: None } - } else { - DebugLoc { file, line, col } - } - } - - pub fn create_debug_loc(&self, scope: &'ll DIScope, span: Span) -> &'ll Value { - let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); - - unsafe { - llvm::LLVMRustDIBuilderCreateDebugLocation( - debug_context(self).llcontext, - line.unwrap_or(UNKNOWN_LINE_NUMBER), - col.unwrap_or(UNKNOWN_COLUMN_NUMBER), - scope, - None, - ) - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4c1fee0106a..daceda20097 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -5,8 +5,9 @@ use rustc_codegen_ssa::coverageinfo::map as coverage_map; use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DINameSpace, DISPFlags, DIScope, - DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, + DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, + DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, + DebugEmissionKind, }; use libc::{c_char, c_int, c_uint, size_t}; @@ -794,6 +795,7 @@ pub mod debuginfo { pub struct DIBuilder<'a>(InvariantOpaque<'a>); pub type DIDescriptor = Metadata; + pub type DILocation = Metadata; pub type DIScope = DIDescriptor; pub type DIFile = DIScope; pub type DILexicalBlock = DIScope; @@ -1854,7 +1856,7 @@ extern "C" { ScopeLine: c_uint, Flags: DIFlags, SPFlags: DISPFlags, - Fn: &'a Value, + MaybeFn: Option<&'a Value>, TParam: &'a DIArray, Decl: Option<&'a DIDescriptor>, ) -> &'a DISubprogram; @@ -2005,7 +2007,7 @@ extern "C" { VarInfo: &'a DIVariable, AddrOps: *const i64, AddrOpsCount: c_uint, - DL: &'a Value, + DL: &'a DILocation, InsertAtEnd: &'a BasicBlock, ) -> &'a Value; @@ -2093,8 +2095,8 @@ extern "C" { Line: c_uint, Column: c_uint, Scope: &'a DIScope, - InlinedAt: Option<&'a Metadata>, - ) -> &'a Value; + InlinedAt: Option<&'a DILocation>, + ) -> &'a DILocation; pub fn LLVMRustDIBuilderCreateOpDeref() -> i64; pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64; diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 5f820f83a94..b6a0516b8bc 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -173,26 +173,24 @@ pub(super) fn emit_va_arg( // is lacking in some instances, so we should only use it as a fallback. let target = &bx.cx.tcx.sess.target; let arch = &bx.cx.tcx.sess.target.arch; - match (&**arch, target.options.is_like_windows) { + match &**arch { // Windows x86 - ("x86", true) => { + "x86" if target.options.is_like_windows => { emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false) } // Generic x86 - ("x86", _) => { - emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true) - } + "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), // Windows AArch64 - ("aarch64", true) => { + "aarch64" if target.options.is_like_windows => { emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) } - // iOS AArch64 - ("aarch64", _) if target.target_os == "ios" => { + // macOS / iOS AArch64 + "aarch64" if target.options.is_like_osx => { emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) } - ("aarch64", _) => emit_aapcs_va_arg(bx, addr, target_ty), + "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), // Windows x86_64 - ("x86_64", true) => { + "x86_64" if target.options.is_like_windows => { let target_ty_size = bx.cx.size_of(target_ty).bytes(); let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index bec0a84cac0..da4637b1afc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -405,7 +405,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.set_debug_loc(&mut bx, terminator.source_info); // Get the location information. - let location = self.get_caller_location(&mut bx, span).immediate(); + let location = self.get_caller_location(&mut bx, terminator.source_info).immediate(); // Put together the arguments to the panic entry point. let (lang_item, args) = match msg { @@ -442,7 +442,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, intrinsic: Option<Symbol>, instance: Option<Instance<'tcx>>, - span: Span, + source_info: mir::SourceInfo, destination: &Option<(mir::Place<'tcx>, mir::BasicBlock)>, cleanup: Option<mir::BasicBlock>, ) -> bool { @@ -484,11 +484,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } }); let msg = bx.const_str(Symbol::intern(&msg_str)); - let location = self.get_caller_location(bx, span).immediate(); + let location = self.get_caller_location(bx, source_info).immediate(); // Obtain the panic entry point. // FIXME: dedup this with `codegen_assert_terminator` above. - let def_id = common::langcall(bx.tcx(), Some(span), "", LangItem::Panic); + let def_id = + common::langcall(bx.tcx(), Some(source_info.span), "", LangItem::Panic); let instance = ty::Instance::mono(bx.tcx(), def_id); let fn_abi = FnAbi::of_instance(bx, instance, &[]); let llfn = bx.get_fn_addr(instance); @@ -529,7 +530,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { cleanup: Option<mir::BasicBlock>, fn_span: Span, ) { - let span = terminator.source_info.span; + let source_info = terminator.source_info; + let span = source_info.span; + // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar. let callee = self.codegen_operand(&mut bx, func); @@ -606,7 +609,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &mut bx, intrinsic, instance, - span, + source_info, destination, cleanup, ) { @@ -627,7 +630,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if intrinsic == Some(sym::caller_location) { if let Some((_, target)) = destination.as_ref() { - let location = self.get_caller_location(&mut bx, fn_span); + let location = self + .get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); if let ReturnDest::IndirectOperand(tmp, _) = ret_dest { location.val.store(&mut bx, tmp); @@ -686,7 +690,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &fn_abi, &args, dest, - terminator.source_info.span, + span, ); if let ReturnDest::IndirectOperand(dst, _) = ret_dest { @@ -793,7 +797,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args.len() + 1, "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR", ); - let location = self.get_caller_location(&mut bx, fn_span); + let location = + self.get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info }); debug!( "codegen_call_terminator({:?}): location={:?} (fn_span {:?})", terminator, location, fn_span @@ -1179,17 +1184,49 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - fn get_caller_location(&mut self, bx: &mut Bx, span: Span) -> OperandRef<'tcx, Bx::Value> { - self.caller_location.unwrap_or_else(|| { + fn get_caller_location( + &mut self, + bx: &mut Bx, + mut source_info: mir::SourceInfo, + ) -> OperandRef<'tcx, Bx::Value> { + let tcx = bx.tcx(); + + let mut span_to_caller_location = |span: Span| { let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span); - let caller = bx.tcx().sess.source_map().lookup_char_pos(topmost.lo()); - let const_loc = bx.tcx().const_caller_location(( + let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo()); + let const_loc = tcx.const_caller_location(( Symbol::intern(&caller.file.name.to_string()), caller.line as u32, caller.col_display as u32 + 1, )); OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty()) - }) + }; + + // Walk up the `SourceScope`s, in case some of them are from MIR inlining. + // If so, the starting `source_info.span` is in the innermost inlined + // function, and will be replaced with outer callsite spans as long + // as the inlined functions were `#[track_caller]`. + loop { + let scope_data = &self.mir.source_scopes[source_info.scope]; + + if let Some((callee, callsite_span)) = scope_data.inlined { + // Stop inside the most nested non-`#[track_caller]` function, + // before ever reaching its caller (which is irrelevant). + if !callee.def.requires_caller_location(tcx) { + return span_to_caller_location(source_info.span); + } + source_info.span = callsite_span; + } + + // Skip past all of the parents with `inlined: None`. + match scope_data.inlined_parent_scope { + Some(parent) => source_info.scope = parent, + None => break, + } + } + + // No inlined `SourceScope`s, or all of them were `#[track_caller]`. + self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span)) } fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> { diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index a2ad27b925c..4811adea9ec 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -10,19 +10,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let Coverage { kind, code_region } = coverage; match kind { CoverageKind::Counter { function_source_hash, id } => { - bx.add_counter_region(self.instance, function_source_hash, id, code_region); + if bx.add_counter_region(self.instance, function_source_hash, id, code_region) { + let coverageinfo = bx.tcx().coverageinfo(self.instance.def_id()); - let coverageinfo = bx.tcx().coverageinfo(self.instance.def_id()); - - let fn_name = bx.create_pgo_func_name_var(self.instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); - let id = bx.const_u32(u32::from(id)); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, id, - ); - bx.instrprof_increment(fn_name, hash, num_counters, id); + let fn_name = bx.create_pgo_func_name_var(self.instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let id = bx.const_u32(u32::from(id)); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, id, + ); + bx.instrprof_increment(fn_name, hash, num_counters, id); + } } CoverageKind::Expression { id, lhs, op, rhs } => { bx.add_counter_expression_region(self.instance, id, lhs, op, rhs, code_region); diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 26a646b0293..4e0396a15a6 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -1,5 +1,4 @@ use crate::traits::*; -use rustc_hir::def_id::CrateNum; use rustc_index::vec::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir; @@ -13,9 +12,8 @@ use super::operand::OperandValue; use super::place::PlaceRef; use super::{FunctionCx, LocalRef}; -pub struct FunctionDebugContext<D> { - pub scopes: IndexVec<mir::SourceScope, DebugScope<D>>, - pub defining_crate: CrateNum, +pub struct FunctionDebugContext<S, L> { + pub scopes: IndexVec<mir::SourceScope, DebugScope<S, L>>, } #[derive(Copy, Clone)] @@ -38,77 +36,84 @@ pub struct PerLocalVarDebugInfo<'tcx, D> { } #[derive(Clone, Copy, Debug)] -pub struct DebugScope<D> { - pub scope_metadata: Option<D>, +pub struct DebugScope<S, L> { + // FIXME(eddyb) this should never be `None`, after initialization. + pub dbg_scope: Option<S>, + + /// Call site location, if this scope was inlined from another function. + pub inlined_at: Option<L>, + // Start and end offsets of the file to which this DIScope belongs. // These are used to quickly determine whether some span refers to the same file. pub file_start_pos: BytePos, pub file_end_pos: BytePos, } -impl<D> DebugScope<D> { - pub fn is_valid(&self) -> bool { - self.scope_metadata.is_some() +impl<'tcx, S: Copy, L: Copy> DebugScope<S, L> { + /// DILocations inherit source file name from the parent DIScope. Due to macro expansions + /// it may so happen that the current span belongs to a different file than the DIScope + /// corresponding to span's containing source scope. If so, we need to create a DIScope + /// "extension" into that file. + pub fn adjust_dbg_scope_for_span<Cx: CodegenMethods<'tcx, DIScope = S, DILocation = L>>( + &self, + cx: &Cx, + span: Span, + ) -> S { + // FIXME(eddyb) this should never be `None`. + let dbg_scope = self + .dbg_scope + .unwrap_or_else(|| bug!("`dbg_scope` is only `None` during initialization")); + + let pos = span.lo(); + if pos < self.file_start_pos || pos >= self.file_end_pos { + let sm = cx.sess().source_map(); + cx.extend_scope_to_file(dbg_scope, &sm.lookup_char_pos(pos).file) + } else { + dbg_scope + } } } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { - let (scope, span) = self.debug_loc(source_info); - bx.set_span(span); - if let Some(scope) = scope { - bx.set_source_location(scope, span); + bx.set_span(source_info.span); + if let Some(dbg_loc) = self.dbg_loc(source_info) { + bx.set_dbg_loc(dbg_loc); } } - pub fn debug_loc(&self, source_info: mir::SourceInfo) -> (Option<Bx::DIScope>, Span) { + fn dbg_loc(&self, source_info: mir::SourceInfo) -> Option<Bx::DILocation> { + let (dbg_scope, inlined_at, span) = self.adjusted_span_and_dbg_scope(source_info)?; + Some(self.cx.dbg_loc(dbg_scope, inlined_at, span)) + } + + fn adjusted_span_and_dbg_scope( + &self, + source_info: mir::SourceInfo, + ) -> Option<(Bx::DIScope, Option<Bx::DILocation>, Span)> { + let span = self.adjust_span_for_debugging(source_info.span); + let scope = &self.debug_context.as_ref()?.scopes[source_info.scope]; + Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span)) + } + + /// In order to have a good line stepping behavior in debugger, we overwrite debug + /// locations of macro expansions with that of the outermost expansion site + /// (unless the crate is being compiled with `-Z debug-macros`). + fn adjust_span_for_debugging(&self, mut span: Span) -> Span { // Bail out if debug info emission is not enabled. - match self.debug_context { - None => return (None, source_info.span), - Some(_) => {} + if self.debug_context.is_none() { + return span; } - // In order to have a good line stepping behavior in debugger, we overwrite debug - // locations of macro expansions with that of the outermost expansion site - // (unless the crate is being compiled with `-Z debug-macros`). - if !source_info.span.from_expansion() || self.cx.sess().opts.debugging_opts.debug_macros { - let scope = self.scope_metadata_for_loc(source_info.scope, source_info.span.lo()); - (scope, source_info.span) - } else { + if span.from_expansion() && !self.cx.sess().opts.debugging_opts.debug_macros { // Walk up the macro expansion chain until we reach a non-expanded span. // We also stop at the function body level because no line stepping can occur // at the level above that. - let span = rustc_span::hygiene::walk_chain(source_info.span, self.mir.span.ctxt()); - let scope = self.scope_metadata_for_loc(source_info.scope, span.lo()); // Use span of the outermost expansion site, while keeping the original lexical scope. - (scope, span) + span = rustc_span::hygiene::walk_chain(span, self.mir.span.ctxt()); } - } - // DILocations inherit source file name from the parent DIScope. Due to macro expansions - // it may so happen that the current span belongs to a different file than the DIScope - // corresponding to span's containing source scope. If so, we need to create a DIScope - // "extension" into that file. - fn scope_metadata_for_loc( - &self, - scope_id: mir::SourceScope, - pos: BytePos, - ) -> Option<Bx::DIScope> { - let debug_context = self.debug_context.as_ref()?; - let scope_metadata = debug_context.scopes[scope_id].scope_metadata; - if pos < debug_context.scopes[scope_id].file_start_pos - || pos >= debug_context.scopes[scope_id].file_end_pos - { - let sm = self.cx.sess().source_map(); - let defining_crate = debug_context.defining_crate; - Some(self.cx.extend_scope_to_file( - scope_metadata.unwrap(), - &sm.lookup_char_pos(pos).file, - defining_crate, - )) - } else { - scope_metadata - } + span } /// Apply debuginfo and/or name, after creating the `alloca` for a local, @@ -149,24 +154,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { let name = kw::Invalid; let decl = &self.mir.local_decls[local]; - let (scope, span) = if full_debug_info { - self.debug_loc(decl.source_info) + let dbg_var = if full_debug_info { + self.adjusted_span_and_dbg_scope(decl.source_info).map( + |(dbg_scope, _, span)| { + // FIXME(eddyb) is this `+ 1` needed at all? + let kind = VariableKind::ArgumentVariable(arg_index + 1); + + let arg_ty = self.monomorphize(&decl.ty); + + self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span) + }, + ) } else { - (None, decl.source_info.span) + None }; - let dbg_var = scope.map(|scope| { - // FIXME(eddyb) is this `+ 1` needed at all? - let kind = VariableKind::ArgumentVariable(arg_index + 1); - - self.cx.create_dbg_var( - self.debug_context.as_ref().unwrap(), - name, - self.monomorphize(&decl.ty), - scope, - kind, - span, - ) - }); Some(PerLocalVarDebugInfo { name, @@ -247,6 +248,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let vars = vars.iter().copied().chain(fallback_var); for var in vars { + let dbg_var = match var.dbg_var { + Some(dbg_var) => dbg_var, + None => continue, + }; + let dbg_loc = match self.dbg_loc(var.source_info) { + Some(dbg_loc) => dbg_loc, + None => continue, + }; + let mut layout = base.layout; let mut direct_offset = Size::ZERO; // FIXME(eddyb) use smallvec here. @@ -283,19 +293,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - let (scope, span) = self.debug_loc(var.source_info); - if let Some(scope) = scope { - if let Some(dbg_var) = var.dbg_var { - bx.dbg_var_addr( - dbg_var, - scope, - base.llval, - direct_offset, - &indirect_offsets, - span, - ); - } - } + bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets); } } @@ -319,12 +317,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut per_local = IndexVec::from_elem(vec![], &self.mir.local_decls); for var in &self.mir.var_debug_info { - let (scope, span) = if full_debug_info { - self.debug_loc(var.source_info) + let dbg_scope_and_span = if full_debug_info { + self.adjusted_span_and_dbg_scope(var.source_info) } else { - (None, var.source_info.span) + None }; - let dbg_var = scope.map(|scope| { + let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| { let place = var.place; let var_ty = self.monomorphized_place_ty(place.as_ref()); let var_kind = if self.mir.local_kind(place.local) == mir::LocalKind::Arg @@ -340,14 +338,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { VariableKind::LocalVariable }; - self.cx.create_dbg_var( - self.debug_context.as_ref().unwrap(), - var.name, - var_ty, - scope, - var_kind, - span, - ) + self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span) }); per_local[var.place.local].push(PerLocalVarDebugInfo { diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index bff263567bf..84e82e88e8e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -26,7 +26,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { mir: &'tcx mir::Body<'tcx>, - debug_context: Option<FunctionDebugContext<Bx::DIScope>>, + debug_context: Option<FunctionDebugContext<Bx::DIScope, Bx::DILocation>>, llfn: Bx::Function, diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index a8d88a95f7a..000ddf42604 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -63,6 +63,7 @@ const X86_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[ ("bmi1", None), ("bmi2", None), ("cmpxchg16b", Some(sym::cmpxchg16b_target_feature)), + ("ermsb", Some(sym::ermsb_target_feature)), ("f16c", Some(sym::f16c_target_feature)), ("fma", None), ("fxsr", None), diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index 3fb189e1984..b9c555c2eb0 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -34,6 +34,7 @@ pub trait BackendTypes { // FIXME(eddyb) find a common convention for all of the debuginfo-related // names (choose between `Dbg`, `Debug`, `DebugInfo`, `DI` etc.). type DIScope: Copy; + type DILocation: Copy; type DIVariable: Copy; } diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs index b74e4e45901..3b1654f3ad4 100644 --- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs @@ -9,14 +9,18 @@ pub trait CoverageInfoMethods: BackendTypes { pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { fn create_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value; + /// Returns true if the counter was added to the coverage map; false if `-Z instrument-coverage` + /// is not enabled (a coverage map is not being generated). fn add_counter_region( &mut self, instance: Instance<'tcx>, function_source_hash: u64, id: CounterValueReference, region: CodeRegion, - ); + ) -> bool; + /// Returns true if the expression was added to the coverage map; false if + /// `-Z instrument-coverage` is not enabled (a coverage map is not being generated). fn add_counter_expression_region( &mut self, instance: Instance<'tcx>, @@ -25,7 +29,9 @@ pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes { op: Op, rhs: ExpressionOperandId, region: CodeRegion, - ); + ) -> bool; - fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion); + /// Returns true if the region was added to the coverage map; false if `-Z instrument-coverage` + /// is not enabled (a coverage map is not being generated). + fn add_unreachable_region(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool; } diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index 1ee0f489ffc..3e66d711d2e 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -1,6 +1,5 @@ use super::BackendTypes; use crate::mir::debuginfo::{FunctionDebugContext, VariableKind}; -use rustc_hir::def_id::CrateNum; use rustc_middle::mir; use rustc_middle::ty::{Instance, Ty}; use rustc_span::{SourceFile, Span, Symbol}; @@ -19,14 +18,29 @@ pub trait DebugInfoMethods<'tcx>: BackendTypes { instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Function, - mir: &mir::Body<'_>, - ) -> Option<FunctionDebugContext<Self::DIScope>>; + mir: &mir::Body<'tcx>, + ) -> Option<FunctionDebugContext<Self::DIScope, Self::DILocation>>; + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option<Self::Function>, + ) -> Self::DIScope; + + fn dbg_loc( + &self, + scope: Self::DIScope, + inlined_at: Option<Self::DILocation>, + span: Span, + ) -> Self::DILocation; fn extend_scope_to_file( &self, scope_metadata: Self::DIScope, file: &SourceFile, - defining_crate: CrateNum, ) -> Self::DIScope; fn debuginfo_finalize(&self); @@ -34,7 +48,6 @@ pub trait DebugInfoMethods<'tcx>: BackendTypes { // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). fn create_dbg_var( &self, - dbg_context: &FunctionDebugContext<Self::DIScope>, variable_name: Symbol, variable_type: Ty<'tcx>, scope_metadata: Self::DIScope, @@ -49,14 +62,13 @@ pub trait DebugInfoBuilderMethods: BackendTypes { fn dbg_var_addr( &mut self, dbg_var: Self::DIVariable, - scope_metadata: Self::DIScope, + dbg_loc: Self::DILocation, variable_alloca: Self::Value, direct_offset: Size, // NB: each offset implies a deref (i.e. they're steps in a pointer chain). indirect_offsets: &[Size], - span: Span, ); - fn set_source_location(&mut self, scope: Self::DIScope, span: Span); + fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation); fn insert_reference_to_gdb_debug_scripts_section_global(&mut self); fn set_var_name(&mut self, value: Self::Value, name: &str); } diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index 82518b7f0c3..8ada6c10479 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -95,6 +95,7 @@ pub trait HasCodegen<'tcx>: Type = Self::Type, Funclet = Self::Funclet, DIScope = Self::DIScope, + DILocation = Self::DILocation, DIVariable = Self::DIVariable, >; } diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index caaf7c0c3c2..23e689fcae7 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -25,7 +25,7 @@ rustc-hash = "1.1.0" smallvec = { version = "1.0", features = ["union", "may_dangle"] } rustc_index = { path = "../rustc_index", package = "rustc_index" } bitflags = "1.2.1" -measureme = "0.7.1" +measureme = "9.0.0" libc = "0.2" stacker = "0.1.12" tempfile = "3.0.5" diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs index bc3d1ce53ba..5f42d46e285 100644 --- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs +++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs @@ -1,6 +1,7 @@ use super::{DirectedGraph, WithNumNodes, WithStartNode, WithSuccessors}; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; +use std::ops::ControlFlow; #[cfg(test)] mod tests; @@ -86,10 +87,6 @@ where } } -/// Allows searches to terminate early with a value. -// FIXME (#75744): remove the alias once the generics are in a better order and `C=()`. -pub type ControlFlow<T> = std::ops::ControlFlow<(), T>; - /// The status of a node in the depth-first search. /// /// See the documentation of `TriColorDepthFirstSearch` to see how a node's status is updated diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs index 2db8e466e11..486a9ba77b7 100644 --- a/compiler/rustc_data_structures/src/graph/scc/mod.rs +++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs @@ -307,10 +307,7 @@ where fn walk_unvisited_node(&mut self, depth: usize, node: G::Node) -> WalkReturn<S> { debug!("walk_unvisited_node(depth = {:?}, node = {:?})", depth, node); - debug_assert!(match self.node_states[node] { - NodeState::NotVisited => true, - _ => false, - }); + debug_assert!(matches!(self.node_states[node], NodeState::NotVisited)); // Push `node` onto the stack. self.node_states[node] = NodeState::BeingVisited { depth }; diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs index 064467174ca..4ed88878418 100644 --- a/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs @@ -29,7 +29,7 @@ impl<N: Idx> VecGraph<N> { // Create the *edge starts* array. We are iterating over over // the (sorted) edge pairs. We maintain the invariant that the - // length of the `node_starts` arary is enough to store the + // length of the `node_starts` array is enough to store the // current source node -- so when we see that the source node // for an edge is greater than the current length, we grow the // edge-starts array by just enough. diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 9958e5dd5e0..7669b78834c 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -28,6 +28,7 @@ #![feature(const_panic)] #![feature(min_const_generics)] #![feature(once_cell)] +#![feature(maybe_uninit_uninit_array)] #![allow(rustc::default_hash_types)] #[macro_use] diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs index c0193e9fa0c..a5b2df1da5d 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs @@ -149,8 +149,8 @@ pub struct ObligationForest<O: ForestObligation> { /// comments in `process_obligation` for details. active_cache: FxHashMap<O::CacheKey, usize>, - /// A vector reused in compress(), to avoid allocating new vectors. - node_rewrites: Vec<usize>, + /// A vector reused in compress() and find_cycles_from_node(), to avoid allocating new vectors. + reused_node_vec: Vec<usize>, obligation_tree_id_generator: ObligationTreeIdGenerator, @@ -251,12 +251,22 @@ enum NodeState { Error, } +/// This trait allows us to have two different Outcome types: +/// - the normal one that does as little as possible +/// - one for tests that does some additional work and checking +pub trait OutcomeTrait { + type Error; + type Obligation; + + fn new() -> Self; + fn mark_not_stalled(&mut self); + fn is_stalled(&self) -> bool; + fn record_completed(&mut self, outcome: &Self::Obligation); + fn record_error(&mut self, error: Self::Error); +} + #[derive(Debug)] pub struct Outcome<O, E> { - /// Obligations that were completely evaluated, including all - /// (transitive) subobligations. Only computed if requested. - pub completed: Option<Vec<O>>, - /// Backtrace of obligations that were found to be in error. pub errors: Vec<Error<O, E>>, @@ -269,12 +279,29 @@ pub struct Outcome<O, E> { pub stalled: bool, } -/// Should `process_obligations` compute the `Outcome::completed` field of its -/// result? -#[derive(PartialEq)] -pub enum DoCompleted { - No, - Yes, +impl<O, E> OutcomeTrait for Outcome<O, E> { + type Error = Error<O, E>; + type Obligation = O; + + fn new() -> Self { + Self { stalled: true, errors: vec![] } + } + + fn mark_not_stalled(&mut self) { + self.stalled = false; + } + + fn is_stalled(&self) -> bool { + self.stalled + } + + fn record_completed(&mut self, _outcome: &Self::Obligation) { + // do nothing + } + + fn record_error(&mut self, error: Self::Error) { + self.errors.push(error) + } } #[derive(Debug, PartialEq, Eq)] @@ -289,7 +316,7 @@ impl<O: ForestObligation> ObligationForest<O> { nodes: vec![], done_cache: Default::default(), active_cache: Default::default(), - node_rewrites: vec![], + reused_node_vec: vec![], obligation_tree_id_generator: (0..).map(ObligationTreeId), error_cache: Default::default(), } @@ -363,8 +390,7 @@ impl<O: ForestObligation> ObligationForest<O> { .map(|(index, _node)| Error { error: error.clone(), backtrace: self.error_at(index) }) .collect(); - let successful_obligations = self.compress(DoCompleted::Yes); - assert!(successful_obligations.unwrap().is_empty()); + self.compress(|_| assert!(false)); errors } @@ -392,16 +418,12 @@ impl<O: ForestObligation> ObligationForest<O> { /// be called in a loop until `outcome.stalled` is false. /// /// This _cannot_ be unrolled (presently, at least). - pub fn process_obligations<P>( - &mut self, - processor: &mut P, - do_completed: DoCompleted, - ) -> Outcome<O, P::Error> + pub fn process_obligations<P, OUT>(&mut self, processor: &mut P) -> OUT where P: ObligationProcessor<Obligation = O>, + OUT: OutcomeTrait<Obligation = O, Error = Error<O, P::Error>>, { - let mut errors = vec![]; - let mut stalled = true; + let mut outcome = OUT::new(); // Note that the loop body can append new nodes, and those new nodes // will then be processed by subsequent iterations of the loop. @@ -429,7 +451,7 @@ impl<O: ForestObligation> ObligationForest<O> { } ProcessResult::Changed(children) => { // We are not (yet) stalled. - stalled = false; + outcome.mark_not_stalled(); node.state.set(NodeState::Success); for child in children { @@ -442,28 +464,22 @@ impl<O: ForestObligation> ObligationForest<O> { } } ProcessResult::Error(err) => { - stalled = false; - errors.push(Error { error: err, backtrace: self.error_at(index) }); + outcome.mark_not_stalled(); + outcome.record_error(Error { error: err, backtrace: self.error_at(index) }); } } index += 1; } - if stalled { - // There's no need to perform marking, cycle processing and compression when nothing - // changed. - return Outcome { - completed: if do_completed == DoCompleted::Yes { Some(vec![]) } else { None }, - errors, - stalled, - }; + // There's no need to perform marking, cycle processing and compression when nothing + // changed. + if !outcome.is_stalled() { + self.mark_successes(); + self.process_cycles(processor); + self.compress(|obl| outcome.record_completed(obl)); } - self.mark_successes(); - self.process_cycles(processor); - let completed = self.compress(do_completed); - - Outcome { completed, errors, stalled } + outcome } /// Returns a vector of obligations for `p` and all of its @@ -526,7 +542,6 @@ impl<O: ForestObligation> ObligationForest<O> { let node = &self.nodes[index]; let state = node.state.get(); if state == NodeState::Success { - node.state.set(NodeState::Waiting); // This call site is cold. self.uninlined_mark_dependents_as_waiting(node); } else { @@ -538,17 +553,18 @@ impl<O: ForestObligation> ObligationForest<O> { // This never-inlined function is for the cold call site. #[inline(never)] fn uninlined_mark_dependents_as_waiting(&self, node: &Node<O>) { + // Mark node Waiting in the cold uninlined code instead of the hot inlined + node.state.set(NodeState::Waiting); self.inlined_mark_dependents_as_waiting(node) } /// Report cycles between all `Success` nodes, and convert all `Success` /// nodes to `Done`. This must be called after `mark_successes`. - fn process_cycles<P>(&self, processor: &mut P) + fn process_cycles<P>(&mut self, processor: &mut P) where P: ObligationProcessor<Obligation = O>, { - let mut stack = vec![]; - + let mut stack = std::mem::take(&mut self.reused_node_vec); for (index, node) in self.nodes.iter().enumerate() { // For some benchmarks this state test is extremely hot. It's a win // to handle the no-op cases immediately to avoid the cost of the @@ -559,6 +575,7 @@ impl<O: ForestObligation> ObligationForest<O> { } debug_assert!(stack.is_empty()); + self.reused_node_vec = stack; } fn find_cycles_from_node<P>(&self, stack: &mut Vec<usize>, processor: &mut P, index: usize) @@ -591,13 +608,12 @@ impl<O: ForestObligation> ObligationForest<O> { /// indices and hence invalidates any outstanding indices. `process_cycles` /// must be run beforehand to remove any cycles on `Success` nodes. #[inline(never)] - fn compress(&mut self, do_completed: DoCompleted) -> Option<Vec<O>> { + fn compress(&mut self, mut outcome_cb: impl FnMut(&O)) { let orig_nodes_len = self.nodes.len(); - let mut node_rewrites: Vec<_> = std::mem::take(&mut self.node_rewrites); + let mut node_rewrites: Vec<_> = std::mem::take(&mut self.reused_node_vec); debug_assert!(node_rewrites.is_empty()); node_rewrites.extend(0..orig_nodes_len); let mut dead_nodes = 0; - let mut removed_done_obligations: Vec<O> = vec![]; // Move removable nodes to the end, preserving the order of the // remaining nodes. @@ -627,10 +643,8 @@ impl<O: ForestObligation> ObligationForest<O> { } else { self.done_cache.insert(node.obligation.as_cache_key().clone()); } - if do_completed == DoCompleted::Yes { - // Extract the success stories. - removed_done_obligations.push(node.obligation.clone()); - } + // Extract the success stories. + outcome_cb(&node.obligation); node_rewrites[index] = orig_nodes_len; dead_nodes += 1; } @@ -654,9 +668,7 @@ impl<O: ForestObligation> ObligationForest<O> { } node_rewrites.truncate(0); - self.node_rewrites = node_rewrites; - - if do_completed == DoCompleted::Yes { Some(removed_done_obligations) } else { None } + self.reused_node_vec = node_rewrites; } fn apply_rewrites(&mut self, node_rewrites: &[usize]) { diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs index 01652465eea..371c62c063f 100644 --- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs +++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs @@ -17,6 +17,40 @@ struct ClosureObligationProcessor<OF, BF, O, E> { marker: PhantomData<(O, E)>, } +struct TestOutcome<O, E> { + pub completed: Vec<O>, + pub errors: Vec<Error<O, E>>, + pub stalled: bool, +} + +impl<O, E> OutcomeTrait for TestOutcome<O, E> +where + O: Clone, +{ + type Error = Error<O, E>; + type Obligation = O; + + fn new() -> Self { + Self { errors: vec![], stalled: false, completed: vec![] } + } + + fn mark_not_stalled(&mut self) { + self.stalled = false; + } + + fn is_stalled(&self) -> bool { + self.stalled + } + + fn record_completed(&mut self, outcome: &Self::Obligation) { + self.completed.push(outcome.clone()) + } + + fn record_error(&mut self, error: Self::Error) { + self.errors.push(error) + } +} + #[allow(non_snake_case)] fn C<OF, BF, O>(of: OF, bf: BF) -> ClosureObligationProcessor<OF, BF, O, &'static str> where @@ -65,20 +99,17 @@ fn push_pop() { // A |-> A.1 // |-> A.2 // |-> A.3 - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), - "B" => ProcessResult::Error("B is for broken"), - "C" => ProcessResult::Changed(vec![]), - "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap(), vec!["C"]); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "B" => ProcessResult::Error("B is for broken"), + "C" => ProcessResult::Changed(vec![]), + "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["C"]); assert_eq!(err, vec![Error { error: "B is for broken", backtrace: vec!["B"] }]); // second round: two delays, one success, creating an uneven set of subtasks: @@ -88,60 +119,51 @@ fn push_pop() { // D |-> D.1 // |-> D.2 forest.register_obligation("D"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.1" => ProcessResult::Unchanged, - "A.2" => ProcessResult::Unchanged, - "A.3" => ProcessResult::Changed(vec!["A.3.i"]), - "D" => ProcessResult::Changed(vec!["D.1", "D.2"]), - "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap(), Vec::<&'static str>::new()); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Unchanged, + "A.2" => ProcessResult::Unchanged, + "A.3" => ProcessResult::Changed(vec!["A.3.i"]), + "D" => ProcessResult::Changed(vec!["D.1", "D.2"]), + "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, Vec::<&'static str>::new()); assert_eq!(err, Vec::new()); // third round: ok in A.1 but trigger an error in A.2. Check that it // propagates to A, but not D.1 or D.2. // D |-> D.1 |-> D.1.i // |-> D.2 |-> D.2.i - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.1" => ProcessResult::Changed(vec![]), - "A.2" => ProcessResult::Error("A is for apple"), - "A.3.i" => ProcessResult::Changed(vec![]), - "D.1" => ProcessResult::Changed(vec!["D.1.i"]), - "D.2" => ProcessResult::Changed(vec!["D.2.i"]), - "D.1.i" | "D.2.i" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Changed(vec![]), + "A.2" => ProcessResult::Error("A is for apple"), + "A.3.i" => ProcessResult::Changed(vec![]), + "D.1" => ProcessResult::Changed(vec!["D.1.i"]), + "D.2" => ProcessResult::Changed(vec!["D.2.i"]), + "D.1.i" | "D.2.i" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["A.1", "A.3", "A.3.i"]); assert_eq!(err, vec![Error { error: "A is for apple", backtrace: vec!["A.2", "A"] }]); // fourth round: error in D.1.i - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D.1.i" => ProcessResult::Error("D is for dumb"), - "D.2.i" => ProcessResult::Changed(vec![]), - _ => panic!("unexpected obligation {:?}", obligation), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D.1.i" => ProcessResult::Error("D is for dumb"), + "D.2.i" => ProcessResult::Changed(vec![]), + _ => panic!("unexpected obligation {:?}", obligation), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["D.2", "D.2.i"]); assert_eq!(err, vec![Error { error: "D is for dumb", backtrace: vec!["D.1.i", "D.1", "D"] }]); @@ -160,72 +182,60 @@ fn success_in_grandchildren() { let mut forest = ObligationForest::new(); forest.register_obligation("A"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), - "A.1" => ProcessResult::Changed(vec![]), - "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]), - "A.3" => ProcessResult::Changed(vec![]), - "A.2.i" | "A.2.ii" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "A.1" => ProcessResult::Changed(vec![]), + "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]), + "A.3" => ProcessResult::Changed(vec![]), + "A.2.i" | "A.2.ii" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["A.1", "A.3"]); assert!(err.is_empty()); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.2.i" => ProcessResult::Unchanged, - "A.2.ii" => ProcessResult::Changed(vec![]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap(), vec!["A.2.ii"]); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i" => ProcessResult::Unchanged, + "A.2.ii" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["A.2.ii"]); assert!(err.is_empty()); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]), - "A.2.i.a" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert!(ok.unwrap().is_empty()); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]), + "A.2.i.a" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert!(ok.is_empty()); assert!(err.is_empty()); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.2.i.a" => ProcessResult::Changed(vec![]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.2.i.a" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["A", "A.2", "A.2.i", "A.2.i.a"]); assert!(err.is_empty()); - let Outcome { completed: ok, errors: err, .. } = - forest.process_obligations(&mut C(|_| unreachable!(), |_| {}), DoCompleted::Yes); + let TestOutcome { completed: ok, errors: err, .. } = + forest.process_obligations(&mut C(|_| unreachable!(), |_| {})); - assert!(ok.unwrap().is_empty()); + assert!(ok.is_empty()); assert!(err.is_empty()); } @@ -235,18 +245,15 @@ fn to_errors_no_throw() { // yields to correct errors (and does not panic, in particular). let mut forest = ObligationForest::new(); forest.register_obligation("A"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), - "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]), + "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); let errors = forest.to_errors(()); assert_eq!(errors[0].backtrace, vec!["A.1", "A"]); @@ -260,51 +267,42 @@ fn diamond() { // check that diamond dependencies are handled correctly let mut forest = ObligationForest::new(); forest.register_obligation("A"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Changed(vec!["A.1", "A.2"]), - "A.1" | "A.2" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["A.1", "A.2"]), + "A.1" | "A.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A.1" => ProcessResult::Changed(vec!["D"]), - "A.2" => ProcessResult::Changed(vec!["D"]), - "D" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A.1" => ProcessResult::Changed(vec!["D"]), + "A.2" => ProcessResult::Changed(vec!["D"]), + "D" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); let mut d_count = 0; - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D" => { - d_count += 1; - ProcessResult::Changed(vec![]) - } - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => { + d_count += 1; + ProcessResult::Changed(vec![]) + } + _ => unreachable!(), + }, + |_| {}, + )); assert_eq!(d_count, 1); - let mut ok = ok.unwrap(); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["A", "A.1", "A.2", "D"]); assert_eq!(err.len(), 0); @@ -313,51 +311,42 @@ fn diamond() { assert_eq!(errors.len(), 0); forest.register_obligation("A'"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]), - "A'.1" | "A'.2" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]), + "A'.1" | "A'.2" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]), - "A'.2" => ProcessResult::Changed(vec!["D'"]), - "D'" | "A'" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]), + "A'.2" => ProcessResult::Changed(vec!["D'"]), + "D'" | "A'" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); let mut d_count = 0; - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D'" => { - d_count += 1; - ProcessResult::Error("operation failed") - } - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D'" => { + d_count += 1; + ProcessResult::Error("operation failed") + } + _ => unreachable!(), + }, + |_| {}, + )); assert_eq!(d_count, 1); - assert_eq!(ok.unwrap().len(), 0); + assert_eq!(ok.len(), 0); assert_eq!( err, vec![super::Error { error: "operation failed", backtrace: vec!["D'", "A'.1", "A'"] }] @@ -375,35 +364,27 @@ fn done_dependency() { forest.register_obligation("B: Sized"); forest.register_obligation("C: Sized"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A: Sized" | "B: Sized" | "C: Sized" => ProcessResult::Changed(vec![]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A: Sized" | "B: Sized" | "C: Sized" => ProcessResult::Changed(vec![]), + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["A: Sized", "B: Sized", "C: Sized"]); assert_eq!(err.len(), 0); forest.register_obligation("(A,B,C): Sized"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "(A,B,C): Sized" => { - ProcessResult::Changed(vec!["A: Sized", "B: Sized", "C: Sized"]) - } - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap(), vec!["(A,B,C): Sized"]); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "(A,B,C): Sized" => ProcessResult::Changed(vec!["A: Sized", "B: Sized", "C: Sized"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok, vec!["(A,B,C): Sized"]); assert_eq!(err.len(), 0); } @@ -416,64 +397,52 @@ fn orphan() { forest.register_obligation("C1"); forest.register_obligation("C2"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Changed(vec!["D", "E"]), - "B" => ProcessResult::Unchanged, - "C1" => ProcessResult::Changed(vec![]), - "C2" => ProcessResult::Changed(vec![]), - "D" | "E" => ProcessResult::Unchanged, - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - let mut ok = ok.unwrap(); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Changed(vec!["D", "E"]), + "B" => ProcessResult::Unchanged, + "C1" => ProcessResult::Changed(vec![]), + "C2" => ProcessResult::Changed(vec![]), + "D" | "E" => ProcessResult::Unchanged, + _ => unreachable!(), + }, + |_| {}, + )); + let mut ok = ok; ok.sort(); assert_eq!(ok, vec!["C1", "C2"]); assert_eq!(err.len(), 0); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D" | "E" => ProcessResult::Unchanged, - "B" => ProcessResult::Changed(vec!["D"]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" | "E" => ProcessResult::Unchanged, + "B" => ProcessResult::Changed(vec!["D"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err.len(), 0); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D" => ProcessResult::Unchanged, - "E" => ProcessResult::Error("E is for error"), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => ProcessResult::Unchanged, + "E" => ProcessResult::Error("E is for error"), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err, vec![super::Error { error: "E is for error", backtrace: vec!["E", "A"] }]); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "D" => ProcessResult::Error("D is dead"), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "D" => ProcessResult::Error("D is dead"), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err, vec![super::Error { error: "D is dead", backtrace: vec!["D"] }]); let errors = forest.to_errors(()); @@ -487,35 +456,29 @@ fn simultaneous_register_and_error() { forest.register_obligation("A"); forest.register_obligation("B"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Error("An error"), - "B" => ProcessResult::Changed(vec!["A"]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Error("An error"), + "B" => ProcessResult::Changed(vec!["A"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]); let mut forest = ObligationForest::new(); forest.register_obligation("B"); forest.register_obligation("A"); - let Outcome { completed: ok, errors: err, .. } = forest.process_obligations( - &mut C( - |obligation| match *obligation { - "A" => ProcessResult::Error("An error"), - "B" => ProcessResult::Changed(vec!["A"]), - _ => unreachable!(), - }, - |_| {}, - ), - DoCompleted::Yes, - ); - assert_eq!(ok.unwrap().len(), 0); + let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C( + |obligation| match *obligation { + "A" => ProcessResult::Error("An error"), + "B" => ProcessResult::Changed(vec!["A"]), + _ => unreachable!(), + }, + |_| {}, + )); + assert_eq!(ok.len(), 0); assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]); } diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs index 363879cbb1d..e598d7a683d 100644 --- a/compiler/rustc_data_structures/src/profiling.rs +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -94,34 +94,9 @@ use std::process; use std::sync::Arc; use std::time::{Duration, Instant}; -use measureme::{EventId, EventIdBuilder, SerializableString, StringId}; +use measureme::{EventId, EventIdBuilder, Profiler, SerializableString, StringId}; use parking_lot::RwLock; -cfg_if! { - if #[cfg(any(windows, target_os = "wasi"))] { - /// FileSerializationSink is faster on Windows - type SerializationSink = measureme::FileSerializationSink; - } else if #[cfg(target_arch = "wasm32")] { - type SerializationSink = measureme::ByteVecSink; - } else { - /// MmapSerializatioSink is faster on macOS and Linux - type SerializationSink = measureme::MmapSerializationSink; - } -} - -type Profiler = measureme::Profiler<SerializationSink>; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)] -pub enum ProfileCategory { - Parsing, - Expansion, - TypeChecking, - BorrowChecking, - Codegen, - Linking, - Other, -} - bitflags::bitflags! { struct EventFilter: u32 { const GENERIC_ACTIVITIES = 1 << 0; @@ -400,7 +375,7 @@ impl SelfProfiler { output_directory: &Path, crate_name: Option<&str>, event_filters: &Option<Vec<String>>, - ) -> Result<SelfProfiler, Box<dyn Error>> { + ) -> Result<SelfProfiler, Box<dyn Error + Send + Sync>> { fs::create_dir_all(output_directory)?; let crate_name = crate_name.unwrap_or("unknown-crate"); @@ -511,13 +486,13 @@ impl SelfProfiler { self.event_filter_mask.contains(EventFilter::QUERY_KEYS) } - pub fn event_id_builder(&self) -> EventIdBuilder<'_, SerializationSink> { + pub fn event_id_builder(&self) -> EventIdBuilder<'_> { EventIdBuilder::new(&self.profiler) } } #[must_use] -pub struct TimingGuard<'a>(Option<measureme::TimingGuard<'a, SerializationSink>>); +pub struct TimingGuard<'a>(Option<measureme::TimingGuard<'a>>); impl<'a> TimingGuard<'a> { #[inline] diff --git a/compiler/rustc_data_structures/src/sip128.rs b/compiler/rustc_data_structures/src/sip128.rs index 2c4eff618c6..53062b9c20d 100644 --- a/compiler/rustc_data_structures/src/sip128.rs +++ b/compiler/rustc_data_structures/src/sip128.rs @@ -1,21 +1,53 @@ //! This is a copy of `core::hash::sip` adapted to providing 128 bit hashes. -use std::cmp; use std::hash::Hasher; -use std::mem; +use std::mem::{self, MaybeUninit}; use std::ptr; #[cfg(test)] mod tests; +// The SipHash algorithm operates on 8-byte chunks. +const ELEM_SIZE: usize = mem::size_of::<u64>(); + +// Size of the buffer in number of elements, not including the spill. +// +// The selection of this size was guided by rustc-perf benchmark comparisons of +// different buffer sizes. It should be periodically reevaluated as the compiler +// implementation and input characteristics change. +// +// Using the same-sized buffer for everything we hash is a performance versus +// complexity tradeoff. The ideal buffer size, and whether buffering should even +// be used, depends on what is being hashed. It may be worth it to size the +// buffer appropriately (perhaps by making SipHasher128 generic over the buffer +// size) or disable buffering depending on what is being hashed. But at this +// time, we use the same buffer size for everything. +const BUFFER_CAPACITY: usize = 8; + +// Size of the buffer in bytes, not including the spill. +const BUFFER_SIZE: usize = BUFFER_CAPACITY * ELEM_SIZE; + +// Size of the buffer in number of elements, including the spill. +const BUFFER_WITH_SPILL_CAPACITY: usize = BUFFER_CAPACITY + 1; + +// Size of the buffer in bytes, including the spill. +const BUFFER_WITH_SPILL_SIZE: usize = BUFFER_WITH_SPILL_CAPACITY * ELEM_SIZE; + +// Index of the spill element in the buffer. +const BUFFER_SPILL_INDEX: usize = BUFFER_WITH_SPILL_CAPACITY - 1; + #[derive(Debug, Clone)] +#[repr(C)] pub struct SipHasher128 { - k0: u64, - k1: u64, - length: usize, // how many bytes we've processed - state: State, // hash State - tail: u64, // unprocessed bytes le - ntail: usize, // how many bytes in tail are valid + // The access pattern during hashing consists of accesses to `nbuf` and + // `buf` until the buffer is full, followed by accesses to `state` and + // `processed`, and then repetition of that pattern until hashing is done. + // This is the basis for the ordering of fields below. However, in practice + // the cache miss-rate for data access is extremely low regardless of order. + nbuf: usize, // how many bytes in buf are valid + buf: [MaybeUninit<u64>; BUFFER_WITH_SPILL_CAPACITY], // unprocessed bytes le + state: State, // hash State + processed: usize, // how many bytes we've processed } #[derive(Debug, Clone, Copy)] @@ -51,178 +83,328 @@ macro_rules! compress { }}; } -/// Loads an integer of the desired type from a byte stream, in LE order. Uses -/// `copy_nonoverlapping` to let the compiler generate the most efficient way -/// to load it from a possibly unaligned address. -/// -/// Unsafe because: unchecked indexing at i..i+size_of(int_ty) -macro_rules! load_int_le { - ($buf:expr, $i:expr, $int_ty:ident) => {{ - debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len()); - let mut data = 0 as $int_ty; - ptr::copy_nonoverlapping( - $buf.get_unchecked($i), - &mut data as *mut _ as *mut u8, - mem::size_of::<$int_ty>(), - ); - data.to_le() - }}; -} - -/// Loads a u64 using up to 7 bytes of a byte slice. It looks clumsy but the -/// `copy_nonoverlapping` calls that occur (via `load_int_le!`) all have fixed -/// sizes and avoid calling `memcpy`, which is good for speed. -/// -/// Unsafe because: unchecked indexing at start..start+len +// Copies up to 8 bytes from source to destination. This performs better than +// `ptr::copy_nonoverlapping` on microbenchmarks and may perform better on real +// workloads since all of the copies have fixed sizes and avoid calling memcpy. +// +// This is specifically designed for copies of up to 8 bytes, because that's the +// maximum of number bytes needed to fill an 8-byte-sized element on which +// SipHash operates. Note that for variable-sized copies which are known to be +// less than 8 bytes, this function will perform more work than necessary unless +// the compiler is able to optimize the extra work away. #[inline] -unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 { - debug_assert!(len < 8); - let mut i = 0; // current byte index (from LSB) in the output u64 - let mut out = 0; - if i + 3 < len { - out = load_int_le!(buf, start + i, u32) as u64; +unsafe fn copy_nonoverlapping_small(src: *const u8, dst: *mut u8, count: usize) { + debug_assert!(count <= 8); + + if count == 8 { + ptr::copy_nonoverlapping(src, dst, 8); + return; + } + + let mut i = 0; + if i + 3 < count { + ptr::copy_nonoverlapping(src.add(i), dst.add(i), 4); i += 4; } - if i + 1 < len { - out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8); + + if i + 1 < count { + ptr::copy_nonoverlapping(src.add(i), dst.add(i), 2); i += 2 } - if i < len { - out |= (*buf.get_unchecked(start + i) as u64) << (i * 8); + + if i < count { + *dst.add(i) = *src.add(i); i += 1; } - debug_assert_eq!(i, len); - out + + debug_assert_eq!(i, count); } +// # Implementation +// +// This implementation uses buffering to reduce the hashing cost for inputs +// consisting of many small integers. Buffering simplifies the integration of +// integer input--the integer write function typically just appends to the +// buffer with a statically sized write, updates metadata, and returns. +// +// Buffering also prevents alternating between writes that do and do not trigger +// the hashing process. Only when the entire buffer is full do we transition +// into hashing. This allows us to keep the hash state in registers for longer, +// instead of loading and storing it before and after processing each element. +// +// When a write fills the buffer, a buffer processing function is invoked to +// hash all of the buffered input. The buffer processing functions are marked +// `#[inline(never)]` so that they aren't inlined into the append functions, +// which ensures the more frequently called append functions remain inlineable +// and don't include register pushing/popping that would only be made necessary +// by inclusion of the complex buffer processing path which uses those +// registers. +// +// The buffer includes a "spill"--an extra element at the end--which simplifies +// the integer write buffer processing path. The value that fills the buffer can +// be written with a statically sized write that may spill over into the spill. +// After the buffer is processed, the part of the value that spilled over can be +// written from the spill to the beginning of the buffer with another statically +// sized write. This write may copy more bytes than actually spilled over, but +// we maintain the metadata such that any extra copied bytes will be ignored by +// subsequent processing. Due to the static sizes, this scheme performs better +// than copying the exact number of bytes needed into the end and beginning of +// the buffer. +// +// The buffer is uninitialized, which improves performance, but may preclude +// efficient implementation of alternative approaches. The improvement is not so +// large that an alternative approach should be disregarded because it cannot be +// efficiently implemented with an uninitialized buffer. On the other hand, an +// uninitialized buffer may become more important should a larger one be used. +// +// # Platform Dependence +// +// The SipHash algorithm operates on byte sequences. It parses the input stream +// as 8-byte little-endian integers. Therefore, given the same byte sequence, it +// produces the same result on big- and little-endian hardware. +// +// However, the Hasher trait has methods which operate on multi-byte integers. +// How they are converted into byte sequences can be endian-dependent (by using +// native byte order) or independent (by consistently using either LE or BE byte +// order). It can also be `isize` and `usize` size dependent (by using the +// native size), or independent (by converting to a common size), supposing the +// values can be represented in 32 bits. +// +// In order to make `SipHasher128` consistent with `SipHasher` in libstd, we +// choose to do the integer to byte sequence conversion in the platform- +// dependent way. Clients can achieve platform-independent hashing by widening +// `isize` and `usize` integers to 64 bits on 32-bit systems and byte-swapping +// integers on big-endian systems before passing them to the writing functions. +// This causes the input byte sequence to look identical on big- and little- +// endian systems (supposing `isize` and `usize` values can be represented in 32 +// bits), which ensures platform-independent results. impl SipHasher128 { #[inline] pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher128 { - let mut state = SipHasher128 { - k0: key0, - k1: key1, - length: 0, - state: State { v0: 0, v1: 0, v2: 0, v3: 0 }, - tail: 0, - ntail: 0, + let mut hasher = SipHasher128 { + nbuf: 0, + buf: MaybeUninit::uninit_array(), + state: State { + v0: key0 ^ 0x736f6d6570736575, + // The XOR with 0xee is only done on 128-bit algorithm version. + v1: key1 ^ (0x646f72616e646f6d ^ 0xee), + v2: key0 ^ 0x6c7967656e657261, + v3: key1 ^ 0x7465646279746573, + }, + processed: 0, }; - state.reset(); - state + + unsafe { + // Initialize spill because we read from it in `short_write_process_buffer`. + *hasher.buf.get_unchecked_mut(BUFFER_SPILL_INDEX) = MaybeUninit::zeroed(); + } + + hasher } + // A specialized write function for values with size <= 8. #[inline] - fn reset(&mut self) { - self.length = 0; - self.state.v0 = self.k0 ^ 0x736f6d6570736575; - self.state.v1 = self.k1 ^ 0x646f72616e646f6d; - self.state.v2 = self.k0 ^ 0x6c7967656e657261; - self.state.v3 = self.k1 ^ 0x7465646279746573; - self.ntail = 0; - - // This is only done in the 128 bit version: - self.state.v1 ^= 0xee; + fn short_write<T>(&mut self, x: T) { + let size = mem::size_of::<T>(); + let nbuf = self.nbuf; + debug_assert!(size <= 8); + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + size < BUFFER_WITH_SPILL_SIZE); + + if nbuf + size < BUFFER_SIZE { + unsafe { + // The memcpy call is optimized away because the size is known. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + ptr::copy_nonoverlapping(&x as *const _ as *const u8, dst, size); + } + + self.nbuf = nbuf + size; + + return; + } + + unsafe { self.short_write_process_buffer(x) } } - // A specialized write function for values with size <= 8. - // - // The input must be zero-extended to 64-bits by the caller. This extension - // isn't hashed, but the implementation requires it for correctness. + // A specialized write function for values with size <= 8 that should only + // be called when the write would cause the buffer to fill. // - // This function, given the same integer size and value, has the same effect - // on both little- and big-endian hardware. It operates on values without - // depending on their sequence in memory, so is independent of endianness. - // - // However, we want SipHasher128 to be platform-dependent, in order to be - // consistent with the platform-dependent SipHasher in libstd. In other - // words, we want: - // - // - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])` - // - big-endian: `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])` - // - // Therefore, in order to produce endian-dependent results, SipHasher128's - // `write_xxx` Hasher trait methods byte-swap `x` prior to zero-extending. - // - // If clients of SipHasher128 itself want platform-independent results, they - // *also* must byte-swap integer inputs before invoking the `write_xxx` - // methods on big-endian hardware (that is, two byte-swaps must occur--one - // in the client, and one in SipHasher128). Additionally, they must extend - // `usize` and `isize` types to 64 bits on 32-bit systems. - #[inline] - fn short_write<T>(&mut self, _x: T, x: u64) { + // SAFETY: the write of `x` into `self.buf` starting at byte offset + // `self.nbuf` must cause `self.buf` to become fully initialized (and not + // overflow) if it wasn't already. + #[inline(never)] + unsafe fn short_write_process_buffer<T>(&mut self, x: T) { let size = mem::size_of::<T>(); - self.length += size; - - // The original number must be zero-extended, not sign-extended. - debug_assert!(if size < 8 { x >> (8 * size) == 0 } else { true }); - - // The number of bytes needed to fill `self.tail`. - let needed = 8 - self.ntail; - - // SipHash parses the input stream as 8-byte little-endian integers. - // Inputs are put into `self.tail` until 8 bytes of data have been - // collected, and then that word is processed. - // - // For example, imagine that `self.tail` is 0x0000_00EE_DDCC_BBAA, - // `self.ntail` is 5 (because 5 bytes have been put into `self.tail`), - // and `needed` is therefore 3. - // - // - Scenario 1, `self.write_u8(0xFF)`: we have already zero-extended - // the input to 0x0000_0000_0000_00FF. We now left-shift it five - // bytes, giving 0x0000_FF00_0000_0000. We then bitwise-OR that value - // into `self.tail`, resulting in 0x0000_FFEE_DDCC_BBAA. - // (Zero-extension of the original input is critical in this scenario - // because we don't want the high two bytes of `self.tail` to be - // touched by the bitwise-OR.) `self.tail` is not yet full, so we - // return early, after updating `self.ntail` to 6. - // - // - Scenario 2, `self.write_u32(0xIIHH_GGFF)`: we have already - // zero-extended the input to 0x0000_0000_IIHH_GGFF. We now - // left-shift it five bytes, giving 0xHHGG_FF00_0000_0000. We then - // bitwise-OR that value into `self.tail`, resulting in - // 0xHHGG_FFEE_DDCC_BBAA. `self.tail` is now full, and we can use it - // to update `self.state`. (As mentioned above, this assumes a - // little-endian machine; on a big-endian machine we would have - // byte-swapped 0xIIHH_GGFF in the caller, giving 0xFFGG_HHII, and we - // would then end up bitwise-ORing 0xGGHH_II00_0000_0000 into - // `self.tail`). - // - self.tail |= x << (8 * self.ntail); - if size < needed { - self.ntail += size; + let nbuf = self.nbuf; + debug_assert!(size <= 8); + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + size >= BUFFER_SIZE); + debug_assert!(nbuf + size < BUFFER_WITH_SPILL_SIZE); + + // Copy first part of input into end of buffer, possibly into spill + // element. The memcpy call is optimized away because the size is known. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + ptr::copy_nonoverlapping(&x as *const _ as *const u8, dst, size); + + // Process buffer. + for i in 0..BUFFER_CAPACITY { + let elem = self.buf.get_unchecked(i).assume_init().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + } + + // Copy remaining input into start of buffer by copying size - 1 + // elements from spill (at most size - 1 bytes could have overflowed + // into the spill). The memcpy call is optimized away because the size + // is known. And the whole copy is optimized away for size == 1. + let src = self.buf.get_unchecked(BUFFER_SPILL_INDEX) as *const _ as *const u8; + ptr::copy_nonoverlapping(src, self.buf.as_mut_ptr() as *mut u8, size - 1); + + // This function should only be called when the write fills the buffer. + // Therefore, when size == 1, the new `self.nbuf` must be zero. The size + // is statically known, so the branch is optimized away. + self.nbuf = if size == 1 { 0 } else { nbuf + size - BUFFER_SIZE }; + self.processed += BUFFER_SIZE; + } + + // A write function for byte slices. + #[inline] + fn slice_write(&mut self, msg: &[u8]) { + let length = msg.len(); + let nbuf = self.nbuf; + debug_assert!(nbuf < BUFFER_SIZE); + + if nbuf + length < BUFFER_SIZE { + unsafe { + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + + if length <= 8 { + copy_nonoverlapping_small(msg.as_ptr(), dst, length); + } else { + // This memcpy is *not* optimized away. + ptr::copy_nonoverlapping(msg.as_ptr(), dst, length); + } + } + + self.nbuf = nbuf + length; + return; } - // `self.tail` is full, process it. - self.state.v3 ^= self.tail; - Sip24Rounds::c_rounds(&mut self.state); - self.state.v0 ^= self.tail; - - // Continuing scenario 2: we have one byte left over from the input. We - // set `self.ntail` to 1 and `self.tail` to `0x0000_0000_IIHH_GGFF >> - // 8*3`, which is 0x0000_0000_0000_00II. (Or on a big-endian machine - // the prior byte-swapping would leave us with 0x0000_0000_0000_00FF.) - // - // The `if` is needed to avoid shifting by 64 bits, which Rust - // complains about. - self.ntail = size - needed; - self.tail = if needed < 8 { x >> (8 * needed) } else { 0 }; + unsafe { self.slice_write_process_buffer(msg) } + } + + // A write function for byte slices that should only be called when the + // write would cause the buffer to fill. + // + // SAFETY: `self.buf` must be initialized up to the byte offset `self.nbuf`, + // and `msg` must contain enough bytes to initialize the rest of the element + // containing the byte offset `self.nbuf`. + #[inline(never)] + unsafe fn slice_write_process_buffer(&mut self, msg: &[u8]) { + let length = msg.len(); + let nbuf = self.nbuf; + debug_assert!(nbuf < BUFFER_SIZE); + debug_assert!(nbuf + length >= BUFFER_SIZE); + + // Always copy first part of input into current element of buffer. + // This function should only be called when the write fills the buffer, + // so we know that there is enough input to fill the current element. + let valid_in_elem = nbuf % ELEM_SIZE; + let needed_in_elem = ELEM_SIZE - valid_in_elem; + + let src = msg.as_ptr(); + let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf); + copy_nonoverlapping_small(src, dst, needed_in_elem); + + // Process buffer. + + // Using `nbuf / ELEM_SIZE + 1` rather than `(nbuf + needed_in_elem) / + // ELEM_SIZE` to show the compiler that this loop's upper bound is > 0. + // We know that is true, because last step ensured we have a full + // element in the buffer. + let last = nbuf / ELEM_SIZE + 1; + + for i in 0..last { + let elem = self.buf.get_unchecked(i).assume_init().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + } + + // Process the remaining element-sized chunks of input. + let mut processed = needed_in_elem; + let input_left = length - processed; + let elems_left = input_left / ELEM_SIZE; + let extra_bytes_left = input_left % ELEM_SIZE; + + for _ in 0..elems_left { + let elem = (msg.as_ptr().add(processed) as *const u64).read_unaligned().to_le(); + self.state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut self.state); + self.state.v0 ^= elem; + processed += ELEM_SIZE; + } + + // Copy remaining input into start of buffer. + let src = msg.as_ptr().add(processed); + let dst = self.buf.as_mut_ptr() as *mut u8; + copy_nonoverlapping_small(src, dst, extra_bytes_left); + + self.nbuf = extra_bytes_left; + self.processed += nbuf + processed; } #[inline] pub fn finish128(mut self) -> (u64, u64) { - let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail; + debug_assert!(self.nbuf < BUFFER_SIZE); - self.state.v3 ^= b; - Sip24Rounds::c_rounds(&mut self.state); - self.state.v0 ^= b; + // Process full elements in buffer. + let last = self.nbuf / ELEM_SIZE; - self.state.v2 ^= 0xee; - Sip24Rounds::d_rounds(&mut self.state); - let _0 = self.state.v0 ^ self.state.v1 ^ self.state.v2 ^ self.state.v3; + // Since we're consuming self, avoid updating members for a potential + // performance gain. + let mut state = self.state; + + for i in 0..last { + let elem = unsafe { self.buf.get_unchecked(i).assume_init().to_le() }; + state.v3 ^= elem; + Sip24Rounds::c_rounds(&mut state); + state.v0 ^= elem; + } + + // Get remaining partial element. + let elem = if self.nbuf % ELEM_SIZE != 0 { + unsafe { + // Ensure element is initialized by writing zero bytes. At most + // `ELEM_SIZE - 1` are required given the above check. It's safe + // to write this many because we have the spill and we maintain + // `self.nbuf` such that this write will start before the spill. + let dst = (self.buf.as_mut_ptr() as *mut u8).add(self.nbuf); + ptr::write_bytes(dst, 0, ELEM_SIZE - 1); + self.buf.get_unchecked(last).assume_init().to_le() + } + } else { + 0 + }; + + // Finalize the hash. + let length = self.processed + self.nbuf; + let b: u64 = ((length as u64 & 0xff) << 56) | elem; + + state.v3 ^= b; + Sip24Rounds::c_rounds(&mut state); + state.v0 ^= b; + + state.v2 ^= 0xee; + Sip24Rounds::d_rounds(&mut state); + let _0 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3; + + state.v1 ^= 0xdd; + Sip24Rounds::d_rounds(&mut state); + let _1 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3; - self.state.v1 ^= 0xdd; - Sip24Rounds::d_rounds(&mut self.state); - let _1 = self.state.v0 ^ self.state.v1 ^ self.state.v2 ^ self.state.v3; (_0, _1) } } @@ -230,92 +412,57 @@ impl SipHasher128 { impl Hasher for SipHasher128 { #[inline] fn write_u8(&mut self, i: u8) { - self.short_write(i, i as u64); + self.short_write(i); } #[inline] fn write_u16(&mut self, i: u16) { - self.short_write(i, i.to_le() as u64); + self.short_write(i); } #[inline] fn write_u32(&mut self, i: u32) { - self.short_write(i, i.to_le() as u64); + self.short_write(i); } #[inline] fn write_u64(&mut self, i: u64) { - self.short_write(i, i.to_le() as u64); + self.short_write(i); } #[inline] fn write_usize(&mut self, i: usize) { - self.short_write(i, i.to_le() as u64); + self.short_write(i); } #[inline] fn write_i8(&mut self, i: i8) { - self.short_write(i, i as u8 as u64); + self.short_write(i as u8); } #[inline] fn write_i16(&mut self, i: i16) { - self.short_write(i, (i as u16).to_le() as u64); + self.short_write(i as u16); } #[inline] fn write_i32(&mut self, i: i32) { - self.short_write(i, (i as u32).to_le() as u64); + self.short_write(i as u32); } #[inline] fn write_i64(&mut self, i: i64) { - self.short_write(i, (i as u64).to_le() as u64); + self.short_write(i as u64); } #[inline] fn write_isize(&mut self, i: isize) { - self.short_write(i, (i as usize).to_le() as u64); + self.short_write(i as usize); } #[inline] fn write(&mut self, msg: &[u8]) { - let length = msg.len(); - self.length += length; - - let mut needed = 0; - - if self.ntail != 0 { - needed = 8 - self.ntail; - self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << (8 * self.ntail); - if length < needed { - self.ntail += length; - return; - } else { - self.state.v3 ^= self.tail; - Sip24Rounds::c_rounds(&mut self.state); - self.state.v0 ^= self.tail; - self.ntail = 0; - } - } - - // Buffered tail is now flushed, process new input. - let len = length - needed; - let left = len & 0x7; - - let mut i = needed; - while i < len - left { - let mi = unsafe { load_int_le!(msg, i, u64) }; - - self.state.v3 ^= mi; - Sip24Rounds::c_rounds(&mut self.state); - self.state.v0 ^= mi; - - i += 8; - } - - self.tail = unsafe { u8to64_le(msg, i, left) }; - self.ntail = left; + self.slice_write(msg); } fn finish(&self) -> u64 { diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs index 2e2274a7b77..5fe967c4158 100644 --- a/compiler/rustc_data_structures/src/sip128/tests.rs +++ b/compiler/rustc_data_structures/src/sip128/tests.rs @@ -450,3 +450,48 @@ fn test_short_write_works() { assert_eq!(h1_hash, h2_hash); } + +macro_rules! test_fill_buffer { + ($type:ty, $write_method:ident) => {{ + // Test filling and overfilling the buffer from all possible offsets + // for a given integer type and its corresponding write method. + const SIZE: usize = std::mem::size_of::<$type>(); + let input = [42; BUFFER_SIZE]; + let x = 0x01234567_89ABCDEF_76543210_FEDCBA98_u128 as $type; + let x_bytes = &x.to_ne_bytes(); + + for i in 1..=SIZE { + let s = &input[..BUFFER_SIZE - i]; + + let mut h1 = SipHasher128::new_with_keys(7, 13); + h1.write(s); + h1.$write_method(x); + + let mut h2 = SipHasher128::new_with_keys(7, 13); + h2.write(s); + h2.write(x_bytes); + + let h1_hash = h1.finish128(); + let h2_hash = h2.finish128(); + + assert_eq!(h1_hash, h2_hash); + } + }}; +} + +#[test] +fn test_fill_buffer() { + test_fill_buffer!(u8, write_u8); + test_fill_buffer!(u16, write_u16); + test_fill_buffer!(u32, write_u32); + test_fill_buffer!(u64, write_u64); + test_fill_buffer!(u128, write_u128); + test_fill_buffer!(usize, write_usize); + + test_fill_buffer!(i8, write_i8); + test_fill_buffer!(i16, write_i16); + test_fill_buffer!(i32, write_i32); + test_fill_buffer!(i64, write_i64); + test_fill_buffer!(i128, write_i128); + test_fill_buffer!(isize, write_isize); +} diff --git a/compiler/rustc_data_structures/src/sso/map.rs b/compiler/rustc_data_structures/src/sso/map.rs index fa510e58314..fe8ae7abf98 100644 --- a/compiler/rustc_data_structures/src/sso/map.rs +++ b/compiler/rustc_data_structures/src/sso/map.rs @@ -395,7 +395,7 @@ where V: Copy, { fn extend<T: IntoIterator<Item = (&'a K, &'a V)>>(&mut self, iter: T) { - self.extend(iter.into_iter().map(|(k, v)| (k.clone(), v.clone()))) + self.extend(iter.into_iter().map(|(k, v)| (*k, *v))) } #[inline] @@ -451,7 +451,7 @@ impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> { fn into_iter(self) -> Self::IntoIter { match self { SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_ref_it)), - SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()), + SsoHashMap::Map(map) => EitherIter::Right(map.iter()), } } } @@ -469,7 +469,7 @@ impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> { fn into_iter(self) -> Self::IntoIter { match self { SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_mut_it)), - SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()), + SsoHashMap::Map(map) => EitherIter::Right(map.iter_mut()), } } } diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index d22f3adfb01..26706cd2b1b 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -512,7 +512,7 @@ impl<T: Clone> Clone for Lock<T> { } } -#[derive(Debug)] +#[derive(Debug, Default)] pub struct RwLock<T>(InnerRwLock<T>); impl<T> RwLock<T> { diff --git a/compiler/rustc_data_structures/src/tagged_ptr.rs b/compiler/rustc_data_structures/src/tagged_ptr.rs index e3839d19365..cd1e12ca450 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr.rs @@ -24,7 +24,7 @@ mod drop; pub use copy::CopyTaggedPtr; pub use drop::TaggedPtr; -/// This describes the pointer type encaspulated by TaggedPtr. +/// This describes the pointer type encapsulated by TaggedPtr. /// /// # Safety /// diff --git a/compiler/rustc_data_structures/src/transitive_relation.rs b/compiler/rustc_data_structures/src/transitive_relation.rs index fe60a99dde0..2e1512b3929 100644 --- a/compiler/rustc_data_structures/src/transitive_relation.rs +++ b/compiler/rustc_data_structures/src/transitive_relation.rs @@ -18,7 +18,7 @@ pub struct TransitiveRelation<T: Eq + Hash> { edges: Vec<Edge>, // This is a cached transitive closure derived from the edges. - // Currently, we build it lazilly and just throw out any existing + // Currently, we build it lazily and just throw out any existing // copy whenever a new edge is added. (The Lock is to permit // the lazy computation.) This is kind of silly, except for the // fact its size is tied to `self.elements.len()`, so I wanted to @@ -255,7 +255,7 @@ impl<T: Clone + Debug + Eq + Hash> TransitiveRelation<T> { // argument is that, after step 2, we know that no element // can reach its successors (in the vector, not the graph). // After step 3, we know that no element can reach any of - // its predecesssors (because of step 2) nor successors + // its predecessors (because of step 2) nor successors // (because we just called `pare_down`) // // This same algorithm is used in `parents` below. diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs index d99e335a77a..f434673c39e 100644 --- a/compiler/rustc_driver/src/lib.rs +++ b/compiler/rustc_driver/src/lib.rs @@ -22,7 +22,7 @@ use rustc_errors::registry::{InvalidErrorCode, Registry}; use rustc_errors::{ErrorReported, PResult}; use rustc_feature::{find_gated_cfg, UnstableFeatures}; use rustc_hir::def_id::LOCAL_CRATE; -use rustc_interface::util::{collect_crate_types, get_builtin_codegen_backend}; +use rustc_interface::util::{self, collect_crate_types, get_builtin_codegen_backend}; use rustc_interface::{interface, Queries}; use rustc_lint::LintStore; use rustc_metadata::locator; @@ -716,7 +716,7 @@ impl RustcDefaultCalls { TargetList => { let mut targets = rustc_target::spec::TARGETS.iter().copied().collect::<Vec<_>>(); - targets.sort(); + targets.sort_unstable(); println!("{}", targets.join("\n")); } Sysroot => println!("{}", sess.sysroot.display()), @@ -793,37 +793,24 @@ impl RustcDefaultCalls { } } -/// Returns a version string such as "0.12.0-dev". -fn release_str() -> Option<&'static str> { - option_env!("CFG_RELEASE") -} - -/// Returns the full SHA1 hash of HEAD of the Git repo from which rustc was built. -fn commit_hash_str() -> Option<&'static str> { - option_env!("CFG_VER_HASH") -} - -/// Returns the "commit date" of HEAD of the Git repo from which rustc was built as a static string. -fn commit_date_str() -> Option<&'static str> { - option_env!("CFG_VER_DATE") -} - /// Prints version information pub fn version(binary: &str, matches: &getopts::Matches) { let verbose = matches.opt_present("verbose"); - println!("{} {}", binary, option_env!("CFG_VERSION").unwrap_or("unknown version")); + println!("{} {}", binary, util::version_str().unwrap_or("unknown version")); if verbose { fn unw(x: Option<&str>) -> &str { x.unwrap_or("unknown") } println!("binary: {}", binary); - println!("commit-hash: {}", unw(commit_hash_str())); - println!("commit-date: {}", unw(commit_date_str())); + println!("commit-hash: {}", unw(util::commit_hash_str())); + println!("commit-date: {}", unw(util::commit_date_str())); println!("host: {}", config::host_triple()); - println!("release: {}", unw(release_str())); - get_builtin_codegen_backend("llvm")().print_version(); + println!("release: {}", unw(util::release_str())); + if cfg!(llvm) { + get_builtin_codegen_backend("llvm")().print_version(); + } } } @@ -1109,7 +1096,9 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> { } if cg_flags.iter().any(|x| *x == "passes=list") { - get_builtin_codegen_backend("llvm")().print_passes(); + if cfg!(llvm) { + get_builtin_codegen_backend("llvm")().print_passes(); + } return None; } @@ -1237,7 +1226,7 @@ pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) { format!("we would appreciate a bug report: {}", bug_report_url).into(), format!( "rustc {} running on {}", - option_env!("CFG_VERSION").unwrap_or("unknown_version"), + util::version_str().unwrap_or("unknown_version"), config::host_triple() ) .into(), diff --git a/compiler/rustc_error_codes/src/error_codes/E0308.md b/compiler/rustc_error_codes/src/error_codes/E0308.md index e2c40f03019..decee630995 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0308.md +++ b/compiler/rustc_error_codes/src/error_codes/E0308.md @@ -1,18 +1,26 @@ Expected type did not match the received type. -Erroneous code example: +Erroneous code examples: ```compile_fail,E0308 -let x: i32 = "I am not a number!"; -// ~~~ ~~~~~~~~~~~~~~~~~~~~ -// | | -// | initializing expression; -// | compiler infers type `&str` -// | -// type `i32` assigned to variable `x` +fn plus_one(x: i32) -> i32 { + x + 1 +} + +plus_one("Not a number"); +// ^^^^^^^^^^^^^^ expected `i32`, found `&str` + +if "Not a bool" { +// ^^^^^^^^^^^^ expected `bool`, found `&str` +} + +let x: f32 = "Not a float"; +// --- ^^^^^^^^^^^^^ expected `f32`, found `&str` +// | +// expected due to this ``` -This error occurs when the compiler is unable to infer the concrete type of a -variable. It can occur in several cases, the most common being a mismatch -between two types: the type the author explicitly assigned, and the type the -compiler inferred. +This error occurs when an expression was used in a place where the compiler +expected an expression of a different type. It can occur in several cases, the +most common being when calling a function and passing an argument which has a +different type than the matching type in the function declaration. diff --git a/compiler/rustc_errors/Cargo.toml b/compiler/rustc_errors/Cargo.toml index e4dbb8db381..5d8ff601e79 100644 --- a/compiler/rustc_errors/Cargo.toml +++ b/compiler/rustc_errors/Cargo.toml @@ -13,6 +13,7 @@ rustc_serialize = { path = "../rustc_serialize" } rustc_span = { path = "../rustc_span" } rustc_macros = { path = "../rustc_macros" } rustc_data_structures = { path = "../rustc_data_structures" } +rustc_lint_defs = { path = "../rustc_lint_defs" } unicode-width = "0.1.4" atty = "0.2" termcolor = "1.0" diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs index 265ba59cccb..6f365c07f6d 100644 --- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs +++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs @@ -72,6 +72,7 @@ fn annotation_type_for_level(level: Level) -> AnnotationType { Level::Help => AnnotationType::Help, // FIXME(#59346): Not sure how to map these two levels Level::Cancelled | Level::FailureNote => AnnotationType::Error, + Level::Allow => panic!("Should not call with Allow"), } } @@ -143,7 +144,8 @@ impl AnnotateSnippetEmitterWriter { title: Some(Annotation { label: Some(&message), id: code.as_ref().map(|c| match c { - DiagnosticId::Error(val) | DiagnosticId::Lint(val) => val.as_str(), + DiagnosticId::Error(val) + | DiagnosticId::Lint { name: val, has_future_breakage: _ } => val.as_str(), }), annotation_type: annotation_type_for_level(*level), }), diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs index 91bfc6296b1..decbf03b9de 100644 --- a/compiler/rustc_errors/src/diagnostic.rs +++ b/compiler/rustc_errors/src/diagnostic.rs @@ -1,10 +1,10 @@ use crate::snippet::Style; -use crate::Applicability; use crate::CodeSuggestion; use crate::Level; use crate::Substitution; use crate::SubstitutionPart; use crate::SuggestionStyle; +use rustc_lint_defs::Applicability; use rustc_span::{MultiSpan, Span, DUMMY_SP}; use std::fmt; @@ -27,7 +27,7 @@ pub struct Diagnostic { #[derive(Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)] pub enum DiagnosticId { Error(String), - Lint(String), + Lint { name: String, has_future_breakage: bool }, } /// For example a note attached to an error. @@ -107,7 +107,14 @@ impl Diagnostic { match self.level { Level::Bug | Level::Fatal | Level::Error | Level::FailureNote => true, - Level::Warning | Level::Note | Level::Help | Level::Cancelled => false, + Level::Warning | Level::Note | Level::Help | Level::Cancelled | Level::Allow => false, + } + } + + pub fn has_future_breakage(&self) -> bool { + match self.code { + Some(DiagnosticId::Lint { has_future_breakage, .. }) => has_future_breakage, + _ => false, } } diff --git a/compiler/rustc_errors/src/diagnostic_builder.rs b/compiler/rustc_errors/src/diagnostic_builder.rs index d1ff6f721c4..56acdf699ef 100644 --- a/compiler/rustc_errors/src/diagnostic_builder.rs +++ b/compiler/rustc_errors/src/diagnostic_builder.rs @@ -1,5 +1,6 @@ -use crate::{Applicability, Handler, Level, StashKey}; use crate::{Diagnostic, DiagnosticId, DiagnosticStyledString}; +use crate::{Handler, Level, StashKey}; +use rustc_lint_defs::Applicability; use rustc_span::{MultiSpan, Span}; use std::fmt::{self, Debug}; diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs index b5155f8e910..2dc7b7e2da3 100644 --- a/compiler/rustc_errors/src/emitter.rs +++ b/compiler/rustc_errors/src/emitter.rs @@ -9,14 +9,15 @@ use Destination::*; +use rustc_lint_defs::FutureBreakage; use rustc_span::source_map::SourceMap; use rustc_span::{MultiSpan, SourceFile, Span}; use crate::snippet::{Annotation, AnnotationType, Line, MultilineAnnotation, Style, StyledString}; use crate::styled_buffer::StyledBuffer; -use crate::{ - pluralize, CodeSuggestion, Diagnostic, DiagnosticId, Level, SubDiagnostic, SuggestionStyle, -}; +use crate::{CodeSuggestion, Diagnostic, DiagnosticId, Level, SubDiagnostic, SuggestionStyle}; + +use rustc_lint_defs::pluralize; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -192,6 +193,8 @@ pub trait Emitter { /// other formats can, and will, simply ignore it. fn emit_artifact_notification(&mut self, _path: &Path, _artifact_type: &str) {} + fn emit_future_breakage_report(&mut self, _diags: Vec<(FutureBreakage, Diagnostic)>) {} + /// Checks if should show explanations about "rustc --explain" fn should_show_explain(&self) -> bool { true @@ -513,7 +516,7 @@ impl Emitter for SilentEmitter { /// Maximum number of lines we will print for a multiline suggestion; arbitrary. /// /// This should be replaced with a more involved mechanism to output multiline suggestions that -/// more closely mimmics the regular diagnostic output, where irrelevant code lines are elided. +/// more closely mimics the regular diagnostic output, where irrelevant code lines are elided. pub const MAX_SUGGESTION_HIGHLIGHT_LINES: usize = 6; /// Maximum number of suggestions to be shown /// @@ -887,7 +890,7 @@ impl EmitterWriter { // or the next are vertical line placeholders. || (annotation.takes_space() // If either this or the next annotation is && next.has_label()) // multiline start/end, move it to a new line - || (annotation.has_label() // so as not to overlap the orizontal lines. + || (annotation.has_label() // so as not to overlap the horizontal lines. && next.takes_space()) || (annotation.takes_space() && next.takes_space()) || (overlaps(next, annotation, l) diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs index 750d36d3d89..d57beb1148a 100644 --- a/compiler/rustc_errors/src/json.rs +++ b/compiler/rustc_errors/src/json.rs @@ -13,8 +13,9 @@ use rustc_span::source_map::{FilePathMapping, SourceMap}; use crate::emitter::{Emitter, HumanReadableErrorType}; use crate::registry::Registry; -use crate::{Applicability, DiagnosticId}; +use crate::DiagnosticId; use crate::{CodeSuggestion, SubDiagnostic}; +use rustc_lint_defs::{Applicability, FutureBreakage}; use rustc_data_structures::sync::Lrc; use rustc_span::hygiene::ExpnData; @@ -131,15 +132,37 @@ impl Emitter for JsonEmitter { } } + fn emit_future_breakage_report(&mut self, diags: Vec<(FutureBreakage, crate::Diagnostic)>) { + let data: Vec<FutureBreakageItem> = diags + .into_iter() + .map(|(breakage, mut diag)| { + if diag.level == crate::Level::Allow { + diag.level = crate::Level::Warning; + } + FutureBreakageItem { + future_breakage_date: breakage.date, + diagnostic: Diagnostic::from_errors_diagnostic(&diag, self), + } + }) + .collect(); + let report = FutureIncompatReport { future_incompat_report: data }; + let result = if self.pretty { + writeln!(&mut self.dst, "{}", as_pretty_json(&report)) + } else { + writeln!(&mut self.dst, "{}", as_json(&report)) + } + .and_then(|_| self.dst.flush()); + if let Err(e) = result { + panic!("failed to print future breakage report: {:?}", e); + } + } + fn source_map(&self) -> Option<&Lrc<SourceMap>> { Some(&self.sm) } fn should_show_explain(&self) -> bool { - match self.json_rendered { - HumanReadableErrorType::Short(_) => false, - _ => true, - } + !matches!(self.json_rendered, HumanReadableErrorType::Short(_)) } } @@ -226,6 +249,17 @@ struct ArtifactNotification<'a> { emit: &'a str, } +#[derive(Encodable)] +struct FutureBreakageItem { + future_breakage_date: Option<&'static str>, + diagnostic: Diagnostic, +} + +#[derive(Encodable)] +struct FutureIncompatReport { + future_incompat_report: Vec<FutureBreakageItem>, +} + impl Diagnostic { fn from_errors_diagnostic(diag: &crate::Diagnostic, je: &JsonEmitter) -> Diagnostic { let sugg = diag.suggestions.iter().map(|sugg| Diagnostic { @@ -435,7 +469,7 @@ impl DiagnosticCode { s.map(|s| { let s = match s { DiagnosticId::Error(s) => s, - DiagnosticId::Lint(s) => s, + DiagnosticId::Lint { name, has_future_breakage: _ } => name, }; let je_result = je.registry.as_ref().map(|registry| registry.try_find_description(&s)).unwrap(); diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index 2e8a4ef327a..593e0d92031 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -21,6 +21,8 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::sync::{self, Lock, Lrc}; use rustc_data_structures::AtomicRef; +use rustc_lint_defs::FutureBreakage; +pub use rustc_lint_defs::{pluralize, Applicability}; use rustc_span::source_map::SourceMap; use rustc_span::{Loc, MultiSpan, Span}; @@ -49,30 +51,6 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; #[cfg(target_arch = "x86_64")] rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16); -/// Indicates the confidence in the correctness of a suggestion. -/// -/// All suggestions are marked with an `Applicability`. Tools use the applicability of a suggestion -/// to determine whether it should be automatically applied or if the user should be consulted -/// before applying the suggestion. -#[derive(Copy, Clone, Debug, PartialEq, Hash, Encodable, Decodable)] -pub enum Applicability { - /// The suggestion is definitely what the user intended. This suggestion should be - /// automatically applied. - MachineApplicable, - - /// The suggestion may be what the user intended, but it is uncertain. The suggestion should - /// result in valid Rust code if it is applied. - MaybeIncorrect, - - /// The suggestion contains placeholders like `(...)` or `{ /* fields */ }`. The suggestion - /// cannot be applied automatically because it will not result in valid Rust code. The user - /// will need to fill in the placeholders. - HasPlaceholders, - - /// The applicability of the suggestion is unknown. - Unspecified, -} - #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)] pub enum SuggestionStyle { /// Hide the suggested code when displaying this suggestion inline. @@ -91,10 +69,7 @@ pub enum SuggestionStyle { impl SuggestionStyle { fn hide_inline(&self) -> bool { - match *self { - SuggestionStyle::ShowCode => false, - _ => true, - } + !matches!(*self, SuggestionStyle::ShowCode) } } @@ -324,6 +299,8 @@ struct HandlerInner { /// The warning count, used for a recap upon finishing deduplicated_warn_count: usize, + + future_breakage_diagnostics: Vec<Diagnostic>, } /// A key denoting where from a diagnostic was stashed. @@ -437,6 +414,7 @@ impl Handler { emitted_diagnostic_codes: Default::default(), emitted_diagnostics: Default::default(), stashed_diagnostics: Default::default(), + future_breakage_diagnostics: Vec::new(), }), } } @@ -506,6 +484,17 @@ impl Handler { result } + /// Construct a builder at the `Allow` level at the given `span` and with the `msg`. + pub fn struct_span_allow( + &self, + span: impl Into<MultiSpan>, + msg: &str, + ) -> DiagnosticBuilder<'_> { + let mut result = self.struct_allow(msg); + result.set_span(span); + result + } + /// Construct a builder at the `Warning` level at the given `span` and with the `msg`. /// Also include a code. pub fn struct_span_warn_with_code( @@ -528,6 +517,11 @@ impl Handler { result } + /// Construct a builder at the `Allow` level with the `msg`. + pub fn struct_allow(&self, msg: &str) -> DiagnosticBuilder<'_> { + DiagnosticBuilder::new(self, Level::Allow, msg) + } + /// Construct a builder at the `Error` level at the given `span` and with the `msg`. pub fn struct_span_err(&self, span: impl Into<MultiSpan>, msg: &str) -> DiagnosticBuilder<'_> { let mut result = self.struct_err(msg); @@ -696,6 +690,10 @@ impl Handler { self.inner.borrow_mut().print_error_count(registry) } + pub fn take_future_breakage_diagnostics(&self) -> Vec<Diagnostic> { + std::mem::take(&mut self.inner.borrow_mut().future_breakage_diagnostics) + } + pub fn abort_if_errors(&self) { self.inner.borrow_mut().abort_if_errors() } @@ -726,6 +724,10 @@ impl Handler { self.inner.borrow_mut().emit_artifact_notification(path, artifact_type) } + pub fn emit_future_breakage_report(&self, diags: Vec<(FutureBreakage, Diagnostic)>) { + self.inner.borrow_mut().emitter.emit_future_breakage_report(diags) + } + pub fn delay_as_bug(&self, diagnostic: Diagnostic) { self.inner.borrow_mut().delay_as_bug(diagnostic) } @@ -751,12 +753,23 @@ impl HandlerInner { return; } + if diagnostic.has_future_breakage() { + self.future_breakage_diagnostics.push(diagnostic.clone()); + } + if diagnostic.level == Warning && !self.flags.can_emit_warnings { + if diagnostic.has_future_breakage() { + (*TRACK_DIAGNOSTICS)(diagnostic); + } return; } (*TRACK_DIAGNOSTICS)(diagnostic); + if diagnostic.level == Allow { + return; + } + if let Some(ref code) = diagnostic.code { self.emitted_diagnostic_codes.insert(code.clone()); } @@ -995,6 +1008,7 @@ pub enum Level { Help, Cancelled, FailureNote, + Allow, } impl fmt::Display for Level { @@ -1020,7 +1034,7 @@ impl Level { spec.set_fg(Some(Color::Cyan)).set_intense(true); } FailureNote => {} - Cancelled => unreachable!(), + Allow | Cancelled => unreachable!(), } spec } @@ -1034,22 +1048,55 @@ impl Level { Help => "help", FailureNote => "failure-note", Cancelled => panic!("Shouldn't call on cancelled error"), + Allow => panic!("Shouldn't call on allowed error"), } } pub fn is_failure_note(&self) -> bool { - match *self { - FailureNote => true, - _ => false, - } + matches!(*self, FailureNote) } } -#[macro_export] -macro_rules! pluralize { - ($x:expr) => { - if $x != 1 { "s" } else { "" } +pub fn add_elided_lifetime_in_path_suggestion( + source_map: &SourceMap, + db: &mut DiagnosticBuilder<'_>, + n: usize, + path_span: Span, + incl_angl_brckt: bool, + insertion_span: Span, + anon_lts: String, +) { + let (replace_span, suggestion) = if incl_angl_brckt { + (insertion_span, anon_lts) + } else { + // When possible, prefer a suggestion that replaces the whole + // `Path<T>` expression with `Path<'_, T>`, rather than inserting `'_, ` + // at a point (which makes for an ugly/confusing label) + if let Ok(snippet) = source_map.span_to_snippet(path_span) { + // But our spans can get out of whack due to macros; if the place we think + // we want to insert `'_` isn't even within the path expression's span, we + // should bail out of making any suggestion rather than panicking on a + // subtract-with-overflow or string-slice-out-out-bounds (!) + // FIXME: can we do better? + if insertion_span.lo().0 < path_span.lo().0 { + return; + } + let insertion_index = (insertion_span.lo().0 - path_span.lo().0) as usize; + if insertion_index > snippet.len() { + return; + } + let (before, after) = snippet.split_at(insertion_index); + (path_span, format!("{}{}{}", before, anon_lts, after)) + } else { + (insertion_span, anon_lts) + } }; + db.span_suggestion( + replace_span, + &format!("indicate the anonymous lifetime{}", pluralize!(n)), + suggestion, + Applicability::MachineApplicable, + ); } // Useful type to use with `Result<>` indicate that an error has already diff --git a/compiler/rustc_errors/src/snippet.rs b/compiler/rustc_errors/src/snippet.rs index fae5b94b3a8..dbb2523f286 100644 --- a/compiler/rustc_errors/src/snippet.rs +++ b/compiler/rustc_errors/src/snippet.rs @@ -158,10 +158,7 @@ impl Annotation { pub fn takes_space(&self) -> bool { // Multiline annotations always have to keep vertical space. - match self.annotation_type { - AnnotationType::MultilineStart(_) | AnnotationType::MultilineEnd(_) => true, - _ => false, - } + matches!(self.annotation_type, AnnotationType::MultilineStart(_) | AnnotationType::MultilineEnd(_)) } } diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index dd087ab9150..c124ab64218 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -3,6 +3,8 @@ use rustc_ast::attr::HasAttrs; use rustc_ast::mut_visit::*; use rustc_ast::ptr::P; +use rustc_ast::token::{DelimToken, Token, TokenKind}; +use rustc_ast::tokenstream::{DelimSpan, LazyTokenStream, Spacing, TokenStream, TokenTree}; use rustc_ast::{self as ast, AttrItem, Attribute, MetaItem}; use rustc_attr as attr; use rustc_data_structures::fx::FxHashMap; @@ -289,7 +291,37 @@ impl<'a> StripUnconfigured<'a> { expanded_attrs .into_iter() .flat_map(|(item, span)| { - let attr = attr::mk_attr_from_item(attr.style, item, span); + let orig_tokens = + attr.tokens.as_ref().unwrap_or_else(|| panic!("Missing tokens for {:?}", attr)); + + // We are taking an attribute of the form `#[cfg_attr(pred, attr)]` + // and producing an attribute of the form `#[attr]`. We + // have captured tokens for `attr` itself, but we need to + // synthesize tokens for the wrapper `#` and `[]`, which + // we do below. + + // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token + // for `attr` when we expand it to `#[attr]` + let pound_token = orig_tokens.create_token_stream().trees().next().unwrap(); + if !matches!(pound_token, TokenTree::Token(Token { kind: TokenKind::Pound, .. })) { + panic!("Bad tokens for attribute {:?}", attr); + } + // We don't really have a good span to use for the syntheized `[]` + // in `#[attr]`, so just use the span of the `#` token. + let bracket_group = TokenTree::Delimited( + DelimSpan::from_single(pound_token.span()), + DelimToken::Bracket, + item.tokens + .as_ref() + .unwrap_or_else(|| panic!("Missing tokens for {:?}", item)) + .create_token_stream(), + ); + + let mut attr = attr::mk_attr_from_item(attr.style, item, span); + attr.tokens = Some(LazyTokenStream::new(TokenStream::new(vec![ + (pound_token, Spacing::Alone), + (bracket_group, Spacing::Alone), + ]))); self.process_cfg_attr(attr) }) .collect() diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs index 0b43225a242..3e5762ab992 100644 --- a/compiler/rustc_expand/src/expand.rs +++ b/compiler/rustc_expand/src/expand.rs @@ -1357,7 +1357,8 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { // we'll expand attributes on expressions separately if !stmt.is_expr() { let (attr, derives, after_derive) = if stmt.is_item() { - self.classify_item(&mut stmt) + // FIXME: Handle custom attributes on statements (#15701) + (None, vec![], false) } else { // ignore derives on non-item statements so it falls through // to the unused-attributes lint @@ -1785,6 +1786,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { span: at.span, id: at.id, style: at.style, + tokens: None, }; } else { noop_visit_attribute(at, self) diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs index 9aed307ec93..eb4aab116f0 100644 --- a/compiler/rustc_expand/src/mbe.rs +++ b/compiler/rustc_expand/src/mbe.rs @@ -102,10 +102,7 @@ impl TokenTree { /// Returns `true` if the given token tree is delimited. fn is_delimited(&self) -> bool { - match *self { - TokenTree::Delimited(..) => true, - _ => false, - } + matches!(*self, TokenTree::Delimited(..)) } /// Returns `true` if the given token tree is a token of the given kind. diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs index 6b419dae3f6..91add4f9218 100644 --- a/compiler/rustc_expand/src/mbe/macro_check.rs +++ b/compiler/rustc_expand/src/mbe/macro_check.rs @@ -134,10 +134,7 @@ enum Stack<'a, T> { impl<'a, T> Stack<'a, T> { /// Returns whether a stack is empty. fn is_empty(&self) -> bool { - match *self { - Stack::Empty => true, - _ => false, - } + matches!(*self, Stack::Empty) } /// Returns a new stack with an element of top. diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index f0e6fe39a3c..a074af0189a 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -288,7 +288,8 @@ fn generic_extension<'cx>( // Replace all the tokens for the corresponding positions in the macro, to maintain // proper positions in error reporting, while maintaining the macro_backtrace. if rhs_spans.len() == tts.len() { - tts = tts.map_enumerated(|i, mut tt| { + tts = tts.map_enumerated(|i, tt| { + let mut tt = tt.clone(); let mut sp = rhs_spans[i]; sp = sp.with_ctxt(tt.span().ctxt()); tt.set_span(sp); @@ -1035,17 +1036,16 @@ fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool { /// a fragment specifier (indeed, these fragments can be followed by /// ANYTHING without fear of future compatibility hazards). fn frag_can_be_followed_by_any(kind: NonterminalKind) -> bool { - match kind { + matches!( + kind, NonterminalKind::Item // always terminated by `}` or `;` | NonterminalKind::Block // exactly one token tree | NonterminalKind::Ident // exactly one token tree | NonterminalKind::Literal // exactly one token tree | NonterminalKind::Meta // exactly one token tree | NonterminalKind::Lifetime // exactly one token tree - | NonterminalKind::TT => true, // exactly one token tree - - _ => false, - } + | NonterminalKind::TT // exactly one token tree + ) } enum IsInFollow { diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 0e5c5fe4d44..3d8a4bbff18 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -232,17 +232,19 @@ pub(super) fn transcribe<'a>( // the meta-var. let ident = MacroRulesNormalizedIdent::new(orignal_ident); if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { - if let MatchedNonterminal(ref nt) = cur_matched { - // FIXME #2887: why do we apply a mark when matching a token tree meta-var - // (e.g. `$x:tt`), but not when we are matching any other type of token - // tree? - if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); + if let MatchedNonterminal(nt) = cur_matched { + let token = if let NtTT(tt) = &**nt { + // `tt`s are emitted into the output stream directly as "raw tokens", + // without wrapping them into groups. + tt.clone() } else { + // Other variables are emitted into the output stream as groups with + // `Delimiter::None` to maintain parsing priorities. + // `Interpolated` is currenty used for such groups in rustc parser. marker.visit_span(&mut sp); - let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); - } + TokenTree::token(token::Interpolated(nt.clone()), sp) + }; + result.push(token.into()); } else { // We were unable to descend far enough. This is an error. return Err(cx.struct_span_err( diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index 4c9271a58df..1bc14ae41bf 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -345,10 +345,10 @@ impl<'a, 'b> MutVisitor for PlaceholderExpander<'a, 'b> { fn visit_mod(&mut self, module: &mut ast::Mod) { noop_visit_mod(module, self); - module.items.retain(|item| match item.kind { - ast::ItemKind::MacCall(_) if !self.cx.ecfg.keep_macs => false, // remove macro definitions - _ => true, - }); + // remove macro definitions + module.items.retain( + |item| !matches!(item.kind, ast::ItemKind::MacCall(_) if !self.cx.ecfg.keep_macs), + ); } fn visit_mac(&mut self, _mac: &mut ast::MacCall) { diff --git a/compiler/rustc_feature/src/active.rs b/compiler/rustc_feature/src/active.rs index c13fe2ae280..ad926a810e6 100644 --- a/compiler/rustc_feature/src/active.rs +++ b/compiler/rustc_feature/src/active.rs @@ -210,6 +210,11 @@ declare_features! ( /// it is not on path for eventual stabilization). (active, no_niche, "1.42.0", None, None), + /// Allows using `#[rustc_allow_const_fn_unstable]`. + /// This is an attribute on `const fn` for the same + /// purpose as `#[allow_internal_unstable]`. + (active, rustc_allow_const_fn_unstable, "1.49.0", Some(69399), None), + // no-tracking-issue-end // ------------------------------------------------------------------------- @@ -238,6 +243,7 @@ declare_features! ( (active, rtm_target_feature, "1.35.0", Some(44839), None), (active, f16c_target_feature, "1.36.0", Some(44839), None), (active, riscv_target_feature, "1.45.0", Some(44839), None), + (active, ermsb_target_feature, "1.49.0", Some(44839), None), // ------------------------------------------------------------------------- // feature-group-end: actual feature gates (target features) @@ -601,6 +607,9 @@ declare_features! ( /// Allow anonymous constants from an inline `const` block (active, inline_const, "1.49.0", Some(76001), None), + /// Allows unsized fn parameters. + (active, unsized_fn_params, "1.49.0", Some(48055), None), + // ------------------------------------------------------------------------- // feature-group-end: actual feature gates // ------------------------------------------------------------------------- @@ -622,6 +631,8 @@ pub const INCOMPLETE_FEATURES: &[Symbol] = &[ sym::lazy_normalization_consts, sym::specialization, sym::inline_const, + sym::repr128, + sym::unsized_locals, ]; /// Some features are not allowed to be used together at the same time, if diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 527a49b0538..57ae534590d 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -83,10 +83,7 @@ impl std::fmt::Debug for AttributeGate { impl AttributeGate { fn is_deprecated(&self) -> bool { - match *self { - Self::Gated(Stability::Deprecated(_, _), ..) => true, - _ => false, - } + matches!(*self, Self::Gated(Stability::Deprecated(_, _), ..)) } } @@ -380,6 +377,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ "allow_internal_unstable side-steps feature gating and stability checks", ), gated!( + rustc_allow_const_fn_unstable, AssumedUsed, template!(Word, List: "feat1, feat2, ..."), + "rustc_allow_const_fn_unstable side-steps feature gating and stability checks" + ), + gated!( allow_internal_unsafe, Normal, template!(Word), "allow_internal_unsafe side-steps the unsafe_code lint", ), @@ -598,7 +599,7 @@ pub fn is_builtin_attr_name(name: Symbol) -> bool { BUILTIN_ATTRIBUTE_MAP.get(&name).is_some() } -pub static BUILTIN_ATTRIBUTE_MAP: SyncLazy<FxHashMap<Symbol, &'static BuiltinAttribute>> = +pub static BUILTIN_ATTRIBUTE_MAP: SyncLazy<FxHashMap<Symbol, &BuiltinAttribute>> = SyncLazy::new(|| { let mut map = FxHashMap::default(); for attr in BUILTIN_ATTRIBUTES.iter() { diff --git a/compiler/rustc_hir/src/def.rs b/compiler/rustc_hir/src/def.rs index 62b12542877..193247af584 100644 --- a/compiler/rustc_hir/src/def.rs +++ b/compiler/rustc_hir/src/def.rs @@ -206,8 +206,10 @@ pub enum Res<Id = hir::HirId> { /// ```rust /// impl Foo { fn test() -> [u8; std::mem::size_of::<Self>()] {} } /// ``` + /// We do however allow `Self` in repeat expression even if it is generic to not break code + /// which already works on stable while causing the `const_evaluatable_unchecked` future compat lint. /// - /// FIXME(lazy_normalization_consts): Remove this bodge once this feature is stable. + /// FIXME(lazy_normalization_consts): Remove this bodge once that feature is stable. SelfTy(Option<DefId> /* trait */, Option<(DefId, bool)> /* impl */), ToolMod, // e.g., `rustfmt` in `#[rustfmt::skip]` diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 0ebed5c3480..b9ec18688c5 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -272,10 +272,7 @@ impl GenericArg<'_> { } pub fn is_const(&self) -> bool { - match self { - GenericArg::Const(_) => true, - _ => false, - } + matches!(self, GenericArg::Const(_)) } pub fn descr(&self) -> &'static str { @@ -486,6 +483,8 @@ impl Generics<'hir> { #[derive(HashStable_Generic)] pub enum SyntheticTyParamKind { ImplTrait, + // Created by the `#[rustc_synthetic]` attribute. + FromAttr, } /// A where-clause in a definition. @@ -978,17 +977,11 @@ impl BinOpKind { } pub fn is_lazy(self) -> bool { - match self { - BinOpKind::And | BinOpKind::Or => true, - _ => false, - } + matches!(self, BinOpKind::And | BinOpKind::Or) } pub fn is_shift(self) -> bool { - match self { - BinOpKind::Shl | BinOpKind::Shr => true, - _ => false, - } + matches!(self, BinOpKind::Shl | BinOpKind::Shr) } pub fn is_comparison(self) -> bool { @@ -1068,10 +1061,7 @@ impl UnOp { /// Returns `true` if the unary operator takes its argument by value. pub fn is_by_value(self) -> bool { - match self { - Self::UnNeg | Self::UnNot => true, - _ => false, - } + matches!(self, Self::UnNeg | Self::UnNot) } } @@ -1099,11 +1089,11 @@ pub enum StmtKind<'hir> { Semi(&'hir Expr<'hir>), } -impl StmtKind<'hir> { - pub fn attrs(&self) -> &'hir [Attribute] { +impl<'hir> StmtKind<'hir> { + pub fn attrs(&self, get_item: impl FnOnce(ItemId) -> &'hir Item<'hir>) -> &'hir [Attribute] { match *self { StmtKind::Local(ref l) => &l.attrs, - StmtKind::Item(_) => &[], + StmtKind::Item(ref item_id) => &get_item(*item_id).attrs, StmtKind::Expr(ref e) | StmtKind::Semi(ref e) => &e.attrs, } } @@ -1407,10 +1397,9 @@ impl Expr<'_> { /// on the given expression should be considered a place expression. pub fn is_place_expr(&self, mut allow_projections_from: impl FnMut(&Self) -> bool) -> bool { match self.kind { - ExprKind::Path(QPath::Resolved(_, ref path)) => match path.res { - Res::Local(..) | Res::Def(DefKind::Static, _) | Res::Err => true, - _ => false, - }, + ExprKind::Path(QPath::Resolved(_, ref path)) => { + matches!(path.res, Res::Local(..) | Res::Def(DefKind::Static, _) | Res::Err) + } // Type ascription inherits its place expression kind from its // operand. See: @@ -2202,10 +2191,7 @@ pub enum ImplicitSelfKind { impl ImplicitSelfKind { /// Does this represent an implicit self? pub fn has_implicit_self(&self) -> bool { - match *self { - ImplicitSelfKind::None => false, - _ => true, - } + !matches!(*self, ImplicitSelfKind::None) } } @@ -2235,10 +2221,7 @@ impl Defaultness { } pub fn is_default(&self) -> bool { - match *self { - Defaultness::Default { .. } => true, - _ => false, - } + matches!(*self, Defaultness::Default { .. }) } } @@ -2369,10 +2352,7 @@ pub enum VisibilityKind<'hir> { impl VisibilityKind<'_> { pub fn is_pub(&self) -> bool { - match *self { - VisibilityKind::Public => true, - _ => false, - } + matches!(*self, VisibilityKind::Public) } pub fn is_pub_restricted(&self) -> bool { @@ -2381,15 +2361,6 @@ impl VisibilityKind<'_> { VisibilityKind::Crate(..) | VisibilityKind::Restricted { .. } => true, } } - - pub fn descr(&self) -> &'static str { - match *self { - VisibilityKind::Public => "public", - VisibilityKind::Inherited => "private", - VisibilityKind::Crate(..) => "crate-visible", - VisibilityKind::Restricted { .. } => "restricted", - } - } } #[derive(Debug, HashStable_Generic)] @@ -2509,10 +2480,7 @@ pub struct FnHeader { impl FnHeader { pub fn is_const(&self) -> bool { - match &self.constness { - Constness::Const => true, - _ => false, - } + matches!(&self.constness, Constness::Const) } } diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 5e4c03bec83..3e4eb9eafd7 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -263,6 +263,7 @@ language_item_table! { // is required to define it somewhere. Additionally, there are restrictions on crates that use // a weak lang item, but do not have it defined. Panic, sym::panic, panic_fn, Target::Fn; + PanicStr, sym::panic_str, panic_str, Target::Fn; PanicBoundsCheck, sym::panic_bounds_check, panic_bounds_check_fn, Target::Fn; PanicInfo, sym::panic_info, panic_info, Target::Struct; PanicLocation, sym::panic_location, panic_location, Target::Struct; diff --git a/compiler/rustc_hir/src/pat_util.rs b/compiler/rustc_hir/src/pat_util.rs index c05d3e44423..9e0a6aae242 100644 --- a/compiler/rustc_hir/src/pat_util.rs +++ b/compiler/rustc_hir/src/pat_util.rs @@ -92,10 +92,7 @@ impl hir::Pat<'_> { /// Checks if the pattern contains any patterns that bind something to /// an ident, e.g., `foo`, or `Foo(foo)` or `foo @ Bar(..)`. pub fn contains_bindings(&self) -> bool { - self.satisfies(|p| match p.kind { - PatKind::Binding(..) => true, - _ => false, - }) + self.satisfies(|p| matches!(p.kind, PatKind::Binding(..))) } /// Checks if the pattern satisfies the given predicate on some sub-pattern. diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 4686b4989ae..f7018ae62aa 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -299,13 +299,11 @@ impl<'a> State<'a> { pub fn break_offset_if_not_bol(&mut self, n: usize, off: isize) { if !self.s.is_beginning_of_line() { self.s.break_offset(n, off) - } else { - if off != 0 && self.s.last_token().is_hardbreak_tok() { - // We do something pretty sketchy here: tuck the nonzero - // offset-adjustment we were going to deposit along with the - // break into the previous hardbreak. - self.s.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); - } + } else if off != 0 && self.s.last_token().is_hardbreak_tok() { + // We do something pretty sketchy here: tuck the nonzero + // offset-adjustment we were going to deposit along with the + // break into the previous hardbreak. + self.s.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); } } @@ -1138,9 +1136,7 @@ impl<'a> State<'a> { fn print_expr_anon_const(&mut self, anon_const: &hir::AnonConst) { self.ibox(INDENT_UNIT); self.s.word_space("const"); - self.s.word("{"); self.print_anon_const(anon_const); - self.s.word("}"); self.end() } @@ -1923,10 +1919,7 @@ impl<'a> State<'a> { self.pclose(); } PatKind::Box(ref inner) => { - let is_range_inner = match inner.kind { - PatKind::Range(..) => true, - _ => false, - }; + let is_range_inner = matches!(inner.kind, PatKind::Range(..)); self.s.word("box "); if is_range_inner { self.popen(); @@ -1937,10 +1930,7 @@ impl<'a> State<'a> { } } PatKind::Ref(ref inner, mutbl) => { - let is_range_inner = match inner.kind { - PatKind::Range(..) => true, - _ => false, - }; + let is_range_inner = matches!(inner.kind, PatKind::Range(..)); self.s.word("&"); self.s.word(mutbl.prefix_str()); if is_range_inner { @@ -2437,10 +2427,7 @@ impl<'a> State<'a> { // // Duplicated from `parse::classify`, but adapted for the HIR. fn expr_requires_semi_to_be_stmt(e: &hir::Expr<'_>) -> bool { - match e.kind { - hir::ExprKind::Match(..) | hir::ExprKind::Block(..) | hir::ExprKind::Loop(..) => false, - _ => true, - } + !matches!(e.kind, hir::ExprKind::Match(..) | hir::ExprKind::Block(..) | hir::ExprKind::Loop(..)) } /// This statement requires a semicolon after it. diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs index 795c5a64d26..524efd04cfc 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs @@ -50,10 +50,10 @@ use super::region_constraints::GenericKind; use super::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TypeTrace, ValuePairs}; use crate::infer; -use crate::infer::OriginalQueryValues; use crate::traits::error_reporting::report_object_safety_error; use crate::traits::{ IfExpressionCause, MatchExpressionArmCause, ObligationCause, ObligationCauseCode, + StatementAsExpression, }; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; @@ -64,7 +64,6 @@ use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; use rustc_hir::{Item, ItemKind, Node}; use rustc_middle::ty::error::TypeError; -use rustc_middle::ty::ParamEnvAnd; use rustc_middle::ty::{ self, subst::{Subst, SubstsRef}, @@ -72,6 +71,7 @@ use rustc_middle::ty::{ }; use rustc_span::{BytePos, DesugaringKind, Pos, Span}; use rustc_target::spec::abi; +use std::ops::ControlFlow; use std::{cmp, fmt}; mod note; @@ -619,6 +619,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { scrut_hir_id, opt_suggest_box_span, arm_span, + scrut_span, .. }) => match source { hir::MatchSource::IfLetDesugar { .. } => { @@ -664,25 +665,59 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { Some(ty::error::ExpectedFound { expected, .. }) => expected, _ => last_ty, }); - let msg = "`match` arms have incompatible types"; - err.span_label(cause.span, msg); + let source_map = self.tcx.sess.source_map(); + let mut any_multiline_arm = source_map.is_multiline(arm_span); if prior_arms.len() <= 4 { for sp in prior_arms { + any_multiline_arm |= source_map.is_multiline(*sp); err.span_label(*sp, format!("this is found to be of type `{}`", t)); } } else if let Some(sp) = prior_arms.last() { + any_multiline_arm |= source_map.is_multiline(*sp); err.span_label( *sp, format!("this and all prior arms are found to be of type `{}`", t), ); } - if let Some(sp) = semi_span { - err.span_suggestion_short( - sp, - "consider removing this semicolon", - String::new(), - Applicability::MachineApplicable, - ); + let outer_error_span = if any_multiline_arm { + // Cover just `match` and the scrutinee expression, not + // the entire match body, to reduce diagram noise. + cause.span.shrink_to_lo().to(scrut_span) + } else { + cause.span + }; + let msg = "`match` arms have incompatible types"; + err.span_label(outer_error_span, msg); + if let Some((sp, boxed)) = semi_span { + if let (StatementAsExpression::NeedsBoxing, [.., prior_arm]) = + (boxed, &prior_arms[..]) + { + err.multipart_suggestion( + "consider removing this semicolon and boxing the expressions", + vec![ + (prior_arm.shrink_to_lo(), "Box::new(".to_string()), + (prior_arm.shrink_to_hi(), ")".to_string()), + (arm_span.shrink_to_lo(), "Box::new(".to_string()), + (arm_span.shrink_to_hi(), ")".to_string()), + (sp, String::new()), + ], + Applicability::HasPlaceholders, + ); + } else if matches!(boxed, StatementAsExpression::NeedsBoxing) { + err.span_suggestion_short( + sp, + "consider removing this semicolon and boxing the expressions", + String::new(), + Applicability::MachineApplicable, + ); + } else { + err.span_suggestion_short( + sp, + "consider removing this semicolon", + String::new(), + Applicability::MachineApplicable, + ); + } } if let Some(ret_sp) = opt_suggest_box_span { // Get return type span and point to it. @@ -705,13 +740,27 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { if let Some(sp) = outer { err.span_label(sp, "`if` and `else` have incompatible types"); } - if let Some(sp) = semicolon { - err.span_suggestion_short( - sp, - "consider removing this semicolon", - String::new(), - Applicability::MachineApplicable, - ); + if let Some((sp, boxed)) = semicolon { + if matches!(boxed, StatementAsExpression::NeedsBoxing) { + err.multipart_suggestion( + "consider removing this semicolon and boxing the expression", + vec![ + (then.shrink_to_lo(), "Box::new(".to_string()), + (then.shrink_to_hi(), ")".to_string()), + (else_sp.shrink_to_lo(), "Box::new(".to_string()), + (else_sp.shrink_to_hi(), ")".to_string()), + (sp, String::new()), + ], + Applicability::MachineApplicable, + ); + } else { + err.span_suggestion_short( + sp, + "consider removing this semicolon", + String::new(), + Applicability::MachineApplicable, + ); + } } if let Some(ret_sp) = opt_suggest_box_span { self.suggest_boxing_for_return_impl_trait( @@ -1449,7 +1498,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { } impl<'tcx> ty::fold::TypeVisitor<'tcx> for OpaqueTypesVisitor<'tcx> { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { if let Some((kind, def_id)) = TyCategory::from_ty(t) { let span = self.tcx.def_span(def_id); // Avoid cluttering the output when the "found" and error span overlap: @@ -1590,6 +1639,16 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { Mismatch::Variable(exp_found) => Some(exp_found), Mismatch::Fixed(_) => None, }; + let exp_found = match terr { + // `terr` has more accurate type information than `exp_found` in match expressions. + ty::error::TypeError::Sorts(terr) + if exp_found.map_or(false, |ef| terr.found == ef.found) => + { + Some(*terr) + } + _ => exp_found, + }; + debug!("exp_found {:?} terr {:?}", exp_found, terr); if let Some(exp_found) = exp_found { self.suggest_as_ref_where_appropriate(span, &exp_found, diag); self.suggest_await_on_expect_found(cause, span, &exp_found, diag); @@ -1611,6 +1670,53 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { self.note_error_origin(diag, cause, exp_found); } + pub fn get_impl_future_output_ty(&self, ty: Ty<'tcx>) -> Option<Ty<'tcx>> { + if let ty::Opaque(def_id, substs) = ty.kind() { + let future_trait = self.tcx.require_lang_item(LangItem::Future, None); + // Future::Output + let item_def_id = self + .tcx + .associated_items(future_trait) + .in_definition_order() + .next() + .unwrap() + .def_id; + + let bounds = self.tcx.explicit_item_bounds(*def_id); + + for (predicate, _) in bounds { + let predicate = predicate.subst(self.tcx, substs); + if let ty::PredicateAtom::Projection(projection_predicate) = + predicate.skip_binders() + { + if projection_predicate.projection_ty.item_def_id == item_def_id { + // We don't account for multiple `Future::Output = Ty` contraints. + return Some(projection_predicate.ty); + } + } + } + } + None + } + + /// A possible error is to forget to add `.await` when using futures: + /// + /// ``` + /// async fn make_u32() -> u32 { + /// 22 + /// } + /// + /// fn take_u32(x: u32) {} + /// + /// async fn foo() { + /// let x = make_u32(); + /// take_u32(x); + /// } + /// ``` + /// + /// This routine checks if the found type `T` implements `Future<Output=U>` where `U` is the + /// expected type. If this is the case, and we are inside of an async body, it suggests adding + /// `.await` to the tail of the expression. fn suggest_await_on_expect_found( &self, cause: &ObligationCause<'tcx>, @@ -1620,50 +1726,76 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { ) { debug!( "suggest_await_on_expect_found: exp_span={:?}, expected_ty={:?}, found_ty={:?}", - exp_span, exp_found.expected, exp_found.found + exp_span, exp_found.expected, exp_found.found, ); - if let ty::Opaque(def_id, _) = *exp_found.expected.kind() { - let future_trait = self.tcx.require_lang_item(LangItem::Future, None); - // Future::Output - let item_def_id = self - .tcx - .associated_items(future_trait) - .in_definition_order() - .next() - .unwrap() - .def_id; + if let ObligationCauseCode::CompareImplMethodObligation { .. } = &cause.code { + return; + } - let projection_ty = self.tcx.projection_ty_from_predicates((def_id, item_def_id)); - if let Some(projection_ty) = projection_ty { - let projection_query = self.canonicalize_query( - &ParamEnvAnd { param_env: self.tcx.param_env(def_id), value: projection_ty }, - &mut OriginalQueryValues::default(), - ); - if let Ok(resp) = self.tcx.normalize_projection_ty(projection_query) { - let normalized_ty = resp.value.value.normalized_ty; - debug!("suggest_await_on_expect_found: normalized={:?}", normalized_ty); - if ty::TyS::same_type(normalized_ty, exp_found.found) { - let span = if let ObligationCauseCode::Pattern { - span, - origin_expr: _, - root_ty: _, - } = cause.code - { - // scrutinee's span - span.unwrap_or(exp_span) - } else { - exp_span - }; - diag.span_suggestion_verbose( - span.shrink_to_hi(), - "consider awaiting on the future", - ".await".to_string(), + match ( + self.get_impl_future_output_ty(exp_found.expected), + self.get_impl_future_output_ty(exp_found.found), + ) { + (Some(exp), Some(found)) if ty::TyS::same_type(exp, found) => match &cause.code { + ObligationCauseCode::IfExpression(box IfExpressionCause { then, .. }) => { + diag.multipart_suggestion( + "consider `await`ing on both `Future`s", + vec![ + (then.shrink_to_hi(), ".await".to_string()), + (exp_span.shrink_to_hi(), ".await".to_string()), + ], + Applicability::MaybeIncorrect, + ); + } + ObligationCauseCode::MatchExpressionArm(box MatchExpressionArmCause { + prior_arms, + .. + }) => { + if let [.., arm_span] = &prior_arms[..] { + diag.multipart_suggestion( + "consider `await`ing on both `Future`s", + vec![ + (arm_span.shrink_to_hi(), ".await".to_string()), + (exp_span.shrink_to_hi(), ".await".to_string()), + ], Applicability::MaybeIncorrect, ); + } else { + diag.help("consider `await`ing on both `Future`s"); } } + _ => { + diag.help("consider `await`ing on both `Future`s"); + } + }, + (_, Some(ty)) if ty::TyS::same_type(exp_found.expected, ty) => { + let span = match cause.code { + // scrutinee's span + ObligationCauseCode::Pattern { span: Some(span), .. } => span, + _ => exp_span, + }; + diag.span_suggestion_verbose( + span.shrink_to_hi(), + "consider `await`ing on the `Future`", + ".await".to_string(), + Applicability::MaybeIncorrect, + ); + } + (Some(ty), _) if ty::TyS::same_type(ty, exp_found.found) => { + let span = match cause.code { + // scrutinee's span + ObligationCauseCode::Pattern { span: Some(span), .. } => span, + _ => exp_span, + }; + diag.span_suggestion_verbose( + span.shrink_to_hi(), + "consider `await`ing on the `Future`", + ".await".to_string(), + Applicability::MaybeIncorrect, + ); } + _ => {} } } diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs index 2f3089f1a92..21023a06bb2 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs @@ -91,17 +91,6 @@ impl<'a, 'tcx> Visitor<'tcx> for FindHirNodeVisitor<'a, 'tcx> { if let (None, Some(ty)) = (self.found_local_pattern, self.node_ty_contains_target(local.hir_id)) { - // FIXME: There's a trade-off here - we can either check that our target span - // is contained in `local.span` or not. If we choose to check containment - // we can avoid some spurious suggestions (see #72690), but we lose - // the ability to report on things like: - // - // ``` - // let x = vec![]; - // ``` - // - // because the target span will be in the macro expansion of `vec![]`. - // At present we choose not to check containment. self.found_local_pattern = Some(&*local.pat); self.found_node_ty = Some(ty); } @@ -113,10 +102,8 @@ impl<'a, 'tcx> Visitor<'tcx> for FindHirNodeVisitor<'a, 'tcx> { if let (None, Some(ty)) = (self.found_arg_pattern, self.node_ty_contains_target(param.hir_id)) { - if self.target_span.contains(param.pat.span) { - self.found_arg_pattern = Some(&*param.pat); - self.found_node_ty = Some(ty); - } + self.found_arg_pattern = Some(&*param.pat); + self.found_node_ty = Some(ty); } } intravisit::walk_body(self, body); diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs index 441cfeea20a..df3dbfca01d 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs @@ -15,6 +15,8 @@ use rustc_middle::ty::{self, AssocItemContainer, RegionKind, Ty, TypeFoldable, T use rustc_span::symbol::Ident; use rustc_span::{MultiSpan, Span}; +use std::ops::ControlFlow; + impl<'a, 'tcx> NiceRegionError<'a, 'tcx> { /// Print the error message for lifetime errors when the return type is a static `impl Trait`, /// `dyn Trait` or if a method call on a trait object introduces a static requirement. @@ -39,6 +41,14 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> { ) if **sub_r == RegionKind::ReStatic => { // This is for an implicit `'static` requirement coming from `impl dyn Trait {}`. if let ObligationCauseCode::UnifyReceiver(ctxt) = &cause.code { + // This may have a closure and it would cause ICE + // through `find_param_with_region` (#78262). + let anon_reg_sup = tcx.is_suitable_region(sup_r)?; + let fn_returns = tcx.return_type_impl_or_dyn_traits(anon_reg_sup.def_id); + if fn_returns.is_empty() { + return None; + } + let param = self.find_param_with_region(sup_r, sub_r)?; let lifetime = if sup_r.has_name() { format!("lifetime `{}`", sup_r) @@ -464,13 +474,13 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> { struct TraitObjectVisitor(Vec<DefId>); impl TypeVisitor<'_> for TraitObjectVisitor { - fn visit_ty(&mut self, t: Ty<'_>) -> bool { + fn visit_ty(&mut self, t: Ty<'_>) -> ControlFlow<()> { match t.kind() { ty::Dynamic(preds, RegionKind::ReStatic) => { if let Some(def_id) = preds.principal_def_id() { self.0.push(def_id); } - false + ControlFlow::CONTINUE } _ => t.super_visit_with(self), } diff --git a/compiler/rustc_infer/src/infer/free_regions.rs b/compiler/rustc_infer/src/infer/free_regions.rs index ffe5fb172be..32f73237dd4 100644 --- a/compiler/rustc_infer/src/infer/free_regions.rs +++ b/compiler/rustc_infer/src/infer/free_regions.rs @@ -157,7 +157,7 @@ impl<'tcx> FreeRegionRelations<'tcx> for FreeRegionMap<'tcx> { impl<'a, 'tcx> Lift<'tcx> for FreeRegionMap<'a> { type Lifted = FreeRegionMap<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<FreeRegionMap<'tcx>> { - self.relation.maybe_map(|&fr| tcx.lift(&fr)).map(|relation| FreeRegionMap { relation }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<FreeRegionMap<'tcx>> { + self.relation.maybe_map(|&fr| tcx.lift(fr)).map(|relation| FreeRegionMap { relation }) } } diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index ff7bbf0562f..acded5351f8 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -678,8 +678,6 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { pub fn unsolved_variables(&self) -> Vec<Ty<'tcx>> { let mut inner = self.inner.borrow_mut(); - // FIXME(const_generics): should there be an equivalent function for const variables? - let mut vars: Vec<Ty<'_>> = inner .type_variables() .unsolved_variables() diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs index abdd6edea90..9b2ffc7a920 100644 --- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs +++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs @@ -30,6 +30,7 @@ use rustc_middle::ty::fold::{TypeFoldable, TypeVisitor}; use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation}; use rustc_middle::ty::{self, InferConst, Ty, TyCtxt}; use std::fmt::Debug; +use std::ops::ControlFlow; #[derive(PartialEq)] pub enum NormalizationStrategy { @@ -740,15 +741,15 @@ struct ScopeInstantiator<'me, 'tcx> { } impl<'me, 'tcx> TypeVisitor<'tcx> for ScopeInstantiator<'me, 'tcx> { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &ty::Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &ty::Binder<T>) -> ControlFlow<()> { self.target_index.shift_in(1); t.super_visit_with(self); self.target_index.shift_out(1); - false + ControlFlow::CONTINUE } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { let ScopeInstantiator { bound_region_scope, next_region, .. } = self; match r { @@ -759,7 +760,7 @@ impl<'me, 'tcx> TypeVisitor<'tcx> for ScopeInstantiator<'me, 'tcx> { _ => {} } - false + ControlFlow::CONTINUE } } diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs index 337772d70b8..fe4ba5aa4e8 100644 --- a/compiler/rustc_infer/src/infer/resolve.rs +++ b/compiler/rustc_infer/src/infer/resolve.rs @@ -3,6 +3,8 @@ use super::{FixupError, FixupResult, InferCtxt, Span}; use rustc_middle::ty::fold::{TypeFolder, TypeVisitor}; use rustc_middle::ty::{self, Const, InferConst, Ty, TyCtxt, TypeFoldable}; +use std::ops::ControlFlow; + /////////////////////////////////////////////////////////////////////////// // OPPORTUNISTIC VAR RESOLVER @@ -121,7 +123,7 @@ impl<'a, 'tcx> UnresolvedTypeFinder<'a, 'tcx> { } impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeFinder<'a, 'tcx> { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { let t = self.infcx.shallow_resolve(t); if t.has_infer_types() { if let ty::Infer(infer_ty) = *t.kind() { @@ -143,7 +145,7 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeFinder<'a, 'tcx> { None }; self.first_unresolved = Some((t, ty_var_span)); - true // Halt visiting. + ControlFlow::BREAK } else { // Otherwise, visit its contents. t.super_visit_with(self) @@ -151,7 +153,7 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeFinder<'a, 'tcx> { } else { // All type variables in inference types must already be resolved, // - no need to visit the contents, continue visiting. - false + ControlFlow::CONTINUE } } } diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs index ea9a4661348..3690a88c0d9 100644 --- a/compiler/rustc_infer/src/lib.rs +++ b/compiler/rustc_infer/src/lib.rs @@ -22,6 +22,7 @@ #![feature(never_type)] #![feature(or_patterns)] #![feature(in_band_lifetimes)] +#![feature(control_flow_enum)] #![recursion_limit = "512"] // For rustdoc #[macro_use] diff --git a/compiler/rustc_infer/src/traits/error_reporting/mod.rs b/compiler/rustc_infer/src/traits/error_reporting/mod.rs index f873358ff9f..835f75ec8ef 100644 --- a/compiler/rustc_infer/src/traits/error_reporting/mod.rs +++ b/compiler/rustc_infer/src/traits/error_reporting/mod.rs @@ -2,12 +2,12 @@ use super::ObjectSafetyViolation; use crate::infer::InferCtxt; use rustc_data_structures::fx::FxHashSet; -use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder}; +use rustc_errors::{struct_span_err, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_middle::ty::TyCtxt; use rustc_span::symbol::Symbol; -use rustc_span::Span; +use rustc_span::{MultiSpan, Span}; use std::fmt; impl<'a, 'tcx> InferCtxt<'a, 'tcx> { @@ -54,10 +54,11 @@ pub fn report_object_safety_error( "the trait `{}` cannot be made into an object", trait_str ); - err.span_label(span, format!("the trait `{}` cannot be made into an object", trait_str)); + err.span_label(span, format!("`{}` cannot be made into an object", trait_str)); let mut reported_violations = FxHashSet::default(); - let mut had_span_label = false; + let mut multi_span = vec![]; + let mut messages = vec![]; for violation in violations { if let ObjectSafetyViolation::SizedSelf(sp) = &violation { if !sp.is_empty() { @@ -71,31 +72,37 @@ pub fn report_object_safety_error( let msg = if trait_span.is_none() || spans.is_empty() { format!("the trait cannot be made into an object because {}", violation.error_msg()) } else { - had_span_label = true; format!("...because {}", violation.error_msg()) }; if spans.is_empty() { err.note(&msg); } else { for span in spans { - err.span_label(span, &msg); + multi_span.push(span); + messages.push(msg.clone()); } } - match (trait_span, violation.solution()) { - (Some(_), Some((note, None))) => { - err.help(¬e); - } - (Some(_), Some((note, Some((sugg, span))))) => { - err.span_suggestion(span, ¬e, sugg, Applicability::MachineApplicable); - } + if trait_span.is_some() { // Only provide the help if its a local trait, otherwise it's not actionable. - _ => {} + violation.solution(&mut err); } } } - if let (Some(trait_span), true) = (trait_span, had_span_label) { - err.span_label(trait_span, "this trait cannot be made into an object..."); + let has_multi_span = !multi_span.is_empty(); + let mut note_span = MultiSpan::from_spans(multi_span.clone()); + if let (Some(trait_span), true) = (trait_span, has_multi_span) { + note_span + .push_span_label(trait_span, "this trait cannot be made into an object...".to_string()); } + for (span, msg) in multi_span.into_iter().zip(messages.into_iter()) { + note_span.push_span_label(span, msg); + } + err.span_note( + note_span, + "for a trait to be \"object safe\" it needs to allow building a vtable to allow the call \ + to be resolvable dynamically; for more information visit \ + <https://doc.rust-lang.org/reference/items/traits.html#object-safety>", + ); if tcx.sess.trait_methods_not_found.borrow().contains(&span) { // Avoid emitting error caused by non-existing method (#58734) diff --git a/compiler/rustc_infer/src/traits/structural_impls.rs b/compiler/rustc_infer/src/traits/structural_impls.rs index c48e58c0482..1a1c2637a6f 100644 --- a/compiler/rustc_infer/src/traits/structural_impls.rs +++ b/compiler/rustc_infer/src/traits/structural_impls.rs @@ -4,6 +4,7 @@ use rustc_middle::ty; use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor}; use std::fmt; +use std::ops::ControlFlow; // Structural impls for the structs in `traits`. @@ -68,7 +69,7 @@ impl<'tcx, O: TypeFoldable<'tcx>> TypeFoldable<'tcx> for traits::Obligation<'tcx } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { self.predicate.visit_with(visitor) } } diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index d9c24cc3994..548b6c03daa 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -204,7 +204,10 @@ pub fn register_plugins<'a>( } }); - Ok((krate, Lrc::new(lint_store))) + let lint_store = Lrc::new(lint_store); + sess.init_lint_store(lint_store.clone()); + + Ok((krate, lint_store)) } fn pre_expansion_lint(sess: &Session, lint_store: &LintStore, krate: &ast::Crate) { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 6553d0ecfdb..dfc6f691457 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -477,6 +477,7 @@ fn test_debugging_options_tracking_hash() { untracked!(dump_mir_dir, String::from("abc")); untracked!(dump_mir_exclude_pass_number, true); untracked!(dump_mir_graphviz, true); + untracked!(emit_future_incompat_report, true); untracked!(emit_stack_sizes, true); untracked!(hir_stats, true); untracked!(identify_regions, true); @@ -550,6 +551,7 @@ fn test_debugging_options_tracking_hash() { tracked!(force_overflow_checks, Some(true)); tracked!(force_unstable_if_unmarked, true); tracked!(fuel, Some(("abc".to_string(), 99))); + tracked!(function_sections, Some(false)); tracked!(human_readable_cgu_names, true); tracked!(inline_in_all_cgus, Some(true)); tracked!(insert_sideeffect, true); @@ -572,6 +574,7 @@ fn test_debugging_options_tracking_hash() { tracked!(print_fuel, Some("abc".to_string())); tracked!(profile, true); tracked!(profile_emit, Some(PathBuf::from("abc"))); + tracked!(relax_elf_relocations, Some(true)); tracked!(relro_level, Some(RelroLevel::Full)); tracked!(report_delayed_bugs, true); tracked!(run_dsymutil, false); diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index c1b359c7d0d..46a6c5861d5 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -24,11 +24,13 @@ use rustc_span::source_map::FileLoader; use rustc_span::symbol::{sym, Symbol}; use smallvec::SmallVec; use std::env; +use std::env::consts::{DLL_PREFIX, DLL_SUFFIX}; use std::io::{self, Write}; use std::lazy::SyncOnceCell; use std::mem; use std::ops::DerefMut; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, Once}; #[cfg(not(parallel_compiler))] use std::{panic, thread}; @@ -113,6 +115,11 @@ impl Write for Sink { Ok(()) } } +impl io::LocalOutput for Sink { + fn clone_box(&self) -> Box<dyn io::LocalOutput> { + Box::new(Self(self.0.clone())) + } +} /// Like a `thread::Builder::spawn` followed by a `join()`, but avoids the need /// for `'static` bounds. @@ -238,7 +245,19 @@ pub fn get_codegen_backend(sopts: &config::Options) -> Box<dyn CodegenBackend> { static mut LOAD: fn() -> Box<dyn CodegenBackend> = || unreachable!(); INIT.call_once(|| { - let codegen_name = sopts.debugging_opts.codegen_backend.as_deref().unwrap_or("llvm"); + #[cfg(feature = "llvm")] + const DEFAULT_CODEGEN_BACKEND: &'static str = "llvm"; + + #[cfg(not(feature = "llvm"))] + const DEFAULT_CODEGEN_BACKEND: &'static str = "cranelift"; + + let codegen_name = sopts + .debugging_opts + .codegen_backend + .as_ref() + .map(|name| &name[..]) + .unwrap_or(DEFAULT_CODEGEN_BACKEND); + let backend = match codegen_name { filename if filename.contains('.') => load_backend_from_dylib(filename.as_ref()), codegen_name => get_builtin_codegen_backend(codegen_name), @@ -367,15 +386,102 @@ fn sysroot_candidates() -> Vec<PathBuf> { } pub fn get_builtin_codegen_backend(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> { - #[cfg(feature = "llvm")] - { - if backend_name == "llvm" { - return rustc_codegen_llvm::LlvmCodegenBackend::new; + match backend_name { + #[cfg(feature = "llvm")] + "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new, + _ => get_codegen_sysroot(backend_name), + } +} + +pub fn get_codegen_sysroot(backend_name: &str) -> fn() -> Box<dyn CodegenBackend> { + // For now we only allow this function to be called once as it'll dlopen a + // few things, which seems to work best if we only do that once. In + // general this assertion never trips due to the once guard in `get_codegen_backend`, + // but there's a few manual calls to this function in this file we protect + // against. + static LOADED: AtomicBool = AtomicBool::new(false); + assert!( + !LOADED.fetch_or(true, Ordering::SeqCst), + "cannot load the default codegen backend twice" + ); + + let target = session::config::host_triple(); + let sysroot_candidates = sysroot_candidates(); + + let sysroot = sysroot_candidates + .iter() + .map(|sysroot| { + let libdir = filesearch::relative_target_lib_path(&sysroot, &target); + sysroot.join(libdir).with_file_name("codegen-backends") + }) + .filter(|f| { + info!("codegen backend candidate: {}", f.display()); + f.exists() + }) + .next(); + let sysroot = sysroot.unwrap_or_else(|| { + let candidates = sysroot_candidates + .iter() + .map(|p| p.display().to_string()) + .collect::<Vec<_>>() + .join("\n* "); + let err = format!( + "failed to find a `codegen-backends` folder \ + in the sysroot candidates:\n* {}", + candidates + ); + early_error(ErrorOutputType::default(), &err); + }); + info!("probing {} for a codegen backend", sysroot.display()); + + let d = sysroot.read_dir().unwrap_or_else(|e| { + let err = format!( + "failed to load default codegen backend, couldn't \ + read `{}`: {}", + sysroot.display(), + e + ); + early_error(ErrorOutputType::default(), &err); + }); + + let mut file: Option<PathBuf> = None; + + let expected_name = + format!("rustc_codegen_{}-{}", backend_name, release_str().expect("CFG_RELEASE")); + for entry in d.filter_map(|e| e.ok()) { + let path = entry.path(); + let filename = match path.file_name().and_then(|s| s.to_str()) { + Some(s) => s, + None => continue, + }; + if !(filename.starts_with(DLL_PREFIX) && filename.ends_with(DLL_SUFFIX)) { + continue; + } + let name = &filename[DLL_PREFIX.len()..filename.len() - DLL_SUFFIX.len()]; + if name != expected_name { + continue; } + if let Some(ref prev) = file { + let err = format!( + "duplicate codegen backends found\n\ + first: {}\n\ + second: {}\n\ + ", + prev.display(), + path.display() + ); + early_error(ErrorOutputType::default(), &err); + } + file = Some(path.clone()); } - let err = format!("unsupported builtin codegen backend `{}`", backend_name); - early_error(ErrorOutputType::default(), &err); + match file { + Some(ref s) => load_backend_from_dylib(s), + None => { + let err = format!("unsupported builtin codegen backend `{}`", backend_name); + early_error(ErrorOutputType::default(), &err); + } + } } pub(crate) fn compute_crate_disambiguator(session: &Session) -> CrateDisambiguator { @@ -782,3 +888,23 @@ impl<'a> MutVisitor for ReplaceBodyWithLoop<'a, '_> { noop_visit_mac(mac, self) } } + +/// Returns a version string such as "rustc 1.46.0 (04488afe3 2020-08-24)" +pub fn version_str() -> Option<&'static str> { + option_env!("CFG_VERSION") +} + +/// Returns a version string such as "0.12.0-dev". +pub fn release_str() -> Option<&'static str> { + option_env!("CFG_RELEASE") +} + +/// Returns the full SHA1 hash of HEAD of the Git repo from which rustc was built. +pub fn commit_hash_str() -> Option<&'static str> { + option_env!("CFG_VER_HASH") +} + +/// Returns the "commit date" of HEAD of the Git repo from which rustc was built as a static string. +pub fn commit_date_str() -> Option<&'static str> { + option_env!("CFG_VER_DATE") +} diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index c5b59a041ab..6539419aefb 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -238,9 +238,10 @@ pub fn is_whitespace(c: char) -> bool { // Note that this set is stable (ie, it doesn't change with different // Unicode versions), so it's ok to just hard-code the values. - match c { + matches!( + c, // Usual ASCII suspects - | '\u{0009}' // \t + '\u{0009}' // \t | '\u{000A}' // \n | '\u{000B}' // vertical tab | '\u{000C}' // form feed @@ -257,9 +258,7 @@ pub fn is_whitespace(c: char) -> bool { // Dedicated whitespace characters from Unicode | '\u{2028}' // LINE SEPARATOR | '\u{2029}' // PARAGRAPH SEPARATOR - => true, - _ => false, - } + ) } /// True if `c` is valid as a first character of an identifier. diff --git a/compiler/rustc_lint/src/array_into_iter.rs b/compiler/rustc_lint/src/array_into_iter.rs index e6be082da0e..0b5bd39f7f9 100644 --- a/compiler/rustc_lint/src/array_into_iter.rs +++ b/compiler/rustc_lint/src/array_into_iter.rs @@ -3,7 +3,7 @@ use rustc_errors::Applicability; use rustc_hir as hir; use rustc_middle::ty; use rustc_middle::ty::adjustment::{Adjust, Adjustment}; -use rustc_session::lint::FutureIncompatibleInfo; +use rustc_session::lint::FutureBreakage; use rustc_span::symbol::sym; declare_lint! { @@ -38,6 +38,9 @@ declare_lint! { @future_incompatible = FutureIncompatibleInfo { reference: "issue #66145 <https://github.com/rust-lang/rust/issues/66145>", edition: None, + future_breakage: Some(FutureBreakage { + date: None + }) }; } diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index a964950f1df..c65cf65b1c7 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -44,7 +44,6 @@ use rustc_middle::lint::LintDiagnosticBuilder; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::subst::{GenericArgKind, Subst}; use rustc_middle::ty::{self, layout::LayoutError, Ty, TyCtxt}; -use rustc_session::lint::FutureIncompatibleInfo; use rustc_session::Session; use rustc_span::edition::Edition; use rustc_span::source_map::Spanned; @@ -994,7 +993,8 @@ impl EarlyLintPass for UnusedDocComment { fn check_stmt(&mut self, cx: &EarlyContext<'_>, stmt: &ast::Stmt) { let kind = match stmt.kind { ast::StmtKind::Local(..) => "statements", - ast::StmtKind::Item(..) => "inner items", + // Disabled pending discussion in #78306 + ast::StmtKind::Item(..) => return, // expressions will be reported by `check_expr`. ast::StmtKind::Empty | ast::StmtKind::Semi(_) @@ -1368,10 +1368,9 @@ impl TypeAliasBounds { hir::QPath::TypeRelative(ref ty, _) => { // If this is a type variable, we found a `T::Assoc`. match ty.kind { - hir::TyKind::Path(hir::QPath::Resolved(None, ref path)) => match path.res { - Res::Def(DefKind::TyParam, _) => true, - _ => false, - }, + hir::TyKind::Path(hir::QPath::Resolved(None, ref path)) => { + matches!(path.res, Res::Def(DefKind::TyParam, _)) + } _ => false, } } @@ -2380,10 +2379,9 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue { return Some(InitKind::Zeroed); } else if cx.tcx.is_diagnostic_item(sym::mem_uninitialized, def_id) { return Some(InitKind::Uninit); - } else if cx.tcx.is_diagnostic_item(sym::transmute, def_id) { - if is_zero(&args[0]) { - return Some(InitKind::Zeroed); - } + } else if cx.tcx.is_diagnostic_item(sym::transmute, def_id) && is_zero(&args[0]) + { + return Some(InitKind::Zeroed); } } } else if let hir::ExprKind::MethodCall(_, _, ref args, _) = expr.kind { @@ -2879,7 +2877,7 @@ impl<'tcx> LateLintPass<'tcx> for ClashingExternDeclarations { fn check_foreign_item(&mut self, cx: &LateContext<'tcx>, this_fi: &hir::ForeignItem<'_>) { trace!("ClashingExternDeclarations: check_foreign_item: {:?}", this_fi); if let ForeignItemKind::Fn(..) = this_fi.kind { - let tcx = *&cx.tcx; + let tcx = cx.tcx; if let Some(existing_hid) = self.insert(tcx, this_fi) { let existing_decl_ty = tcx.type_of(tcx.hir().local_def_id(existing_hid)); let this_decl_ty = tcx.type_of(tcx.hir().local_def_id(this_fi.hir_id)); diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs index 48270eb59a0..4cfeb0d968b 100644 --- a/compiler/rustc_lint/src/context.rs +++ b/compiler/rustc_lint/src/context.rs @@ -22,7 +22,7 @@ use rustc_ast as ast; use rustc_ast::util::lev_distance::find_best_match_for_name; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync; -use rustc_errors::{struct_span_err, Applicability}; +use rustc_errors::{add_elided_lifetime_in_path_suggestion, struct_span_err, Applicability}; use rustc_hir as hir; use rustc_hir::def::Res; use rustc_hir::def_id::{CrateNum, DefId}; @@ -33,9 +33,10 @@ use rustc_middle::middle::stability; use rustc_middle::ty::layout::{LayoutError, TyAndLayout}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, print::Printer, subst::GenericArg, Ty, TyCtxt}; -use rustc_session::lint::{add_elided_lifetime_in_path_suggestion, BuiltinLintDiagnostics}; +use rustc_session::lint::BuiltinLintDiagnostics; use rustc_session::lint::{FutureIncompatibleInfo, Level, Lint, LintBuffer, LintId}; use rustc_session::Session; +use rustc_session::SessionLintStore; use rustc_span::{symbol::Symbol, MultiSpan, Span, DUMMY_SP}; use rustc_target::abi::LayoutOf; @@ -69,6 +70,20 @@ pub struct LintStore { lint_groups: FxHashMap<&'static str, LintGroup>, } +impl SessionLintStore for LintStore { + fn name_to_lint(&self, lint_name: &str) -> LintId { + let lints = self + .find_lints(lint_name) + .unwrap_or_else(|_| panic!("Failed to find lint with name `{}`", lint_name)); + + if let &[lint] = lints.as_slice() { + return lint; + } else { + panic!("Found mutliple lints with name `{}`: {:?}", lint_name, lints); + } + } +} + /// The target of the `by_name` map, which accounts for renaming/deprecation. enum TargetLint { /// A direct lint target @@ -543,7 +558,7 @@ pub trait LintContext: Sized { anon_lts, ) => { add_elided_lifetime_in_path_suggestion( - sess, + sess.source_map(), &mut db, n, path_span, diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs index 4c8baa49edf..9aeeb627792 100644 --- a/compiler/rustc_lint/src/early.rs +++ b/compiler/rustc_lint/src/early.rs @@ -18,6 +18,7 @@ use crate::context::{EarlyContext, LintContext, LintStore}; use crate::passes::{EarlyLintPass, EarlyLintPassObject}; use rustc_ast as ast; use rustc_ast::visit as ast_visit; +use rustc_attr::HasAttrs; use rustc_session::lint::{BufferedEarlyLint, LintBuffer, LintPass}; use rustc_session::Session; use rustc_span::symbol::Ident; @@ -119,8 +120,22 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T> } fn visit_stmt(&mut self, s: &'a ast::Stmt) { - run_early_pass!(self, check_stmt, s); - self.check_id(s.id); + // Add the statement's lint attributes to our + // current state when checking the statement itself. + // This allows us to handle attributes like + // `#[allow(unused_doc_comments)]`, which apply to + // sibling attributes on the same target + // + // Note that statements get their attributes from + // the AST struct that they wrap (e.g. an item) + self.with_lint_attrs(s.id, s.attrs(), |cx| { + run_early_pass!(cx, check_stmt, s); + cx.check_id(s.id); + }); + // The visitor for the AST struct wrapped + // by the statement (e.g. `Item`) will call + // `with_lint_attrs`, so do this walk + // outside of the above `with_lint_attrs` call ast_visit::walk_stmt(self, s); } diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs index a6c04fb0b4c..015e1098711 100644 --- a/compiler/rustc_lint/src/late.rs +++ b/compiler/rustc_lint/src/late.rs @@ -174,12 +174,13 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas } fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) { - // statement attributes are actually just attributes on one of - // - item - // - local - // - expression - // so we keep track of lint levels there - lint_callback!(self, check_stmt, s); + let get_item = |id: hir::ItemId| self.context.tcx.hir().item(id.id); + let attrs = &s.kind.attrs(get_item); + // See `EarlyContextAndPass::visit_stmt` for an explanation + // of why we call `walk_stmt` outside of `with_lint_attrs` + self.with_lint_attrs(s.hir_id, attrs, |cx| { + lint_callback!(cx, check_stmt, s); + }); hir_visit::walk_stmt(self, s); } diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs index 222333a578b..f36f598ade2 100644 --- a/compiler/rustc_lint/src/levels.rs +++ b/compiler/rustc_lint/src/levels.rs @@ -562,6 +562,13 @@ impl<'tcx> intravisit::Visitor<'tcx> for LintLevelMapBuilder<'_, 'tcx> { }) } + fn visit_stmt(&mut self, e: &'tcx hir::Stmt<'tcx>) { + // We will call `with_lint_attrs` when we walk + // the `StmtKind`. The outer statement itself doesn't + // define the lint levels. + intravisit::walk_stmt(self, e); + } + fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) { self.with_lint_attrs(e.hir_id, &e.attrs, |builder| { intravisit::walk_expr(builder, e); diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs index 1db59bfc39d..7297a6de420 100644 --- a/compiler/rustc_lint/src/lib.rs +++ b/compiler/rustc_lint/src/lib.rs @@ -35,6 +35,9 @@ #![feature(never_type)] #![feature(nll)] #![feature(or_patterns)] +#![feature(half_open_range_patterns)] +#![feature(exclusive_range_pattern)] +#![feature(control_flow_enum)] #![recursion_limit = "256"] #[macro_use] @@ -49,6 +52,7 @@ mod early; mod internal; mod late; mod levels; +mod methods; mod non_ascii_idents; mod nonstandard_style; mod passes; @@ -73,6 +77,7 @@ use rustc_span::Span; use array_into_iter::ArrayIntoIter; use builtin::*; use internal::*; +use methods::*; use non_ascii_idents::*; use nonstandard_style::*; use redundant_semicolon::*; @@ -160,6 +165,7 @@ macro_rules! late_lint_passes { ArrayIntoIter: ArrayIntoIter, ClashingExternDeclarations: ClashingExternDeclarations::new(), DropTraitConstraints: DropTraitConstraints, + TemporaryCStringAsPtr: TemporaryCStringAsPtr, ] ); }; diff --git a/compiler/rustc_lint/src/methods.rs b/compiler/rustc_lint/src/methods.rs new file mode 100644 index 00000000000..8732845af0c --- /dev/null +++ b/compiler/rustc_lint/src/methods.rs @@ -0,0 +1,106 @@ +use crate::LateContext; +use crate::LateLintPass; +use crate::LintContext; +use rustc_hir::{Expr, ExprKind, PathSegment}; +use rustc_middle::ty; +use rustc_span::{symbol::sym, ExpnKind, Span}; + +declare_lint! { + /// The `temporary_cstring_as_ptr` lint detects getting the inner pointer of + /// a temporary `CString`. + /// + /// ### Example + /// + /// ```rust + /// # #![allow(unused)] + /// # use std::ffi::CString; + /// let c_str = CString::new("foo").unwrap().as_ptr(); + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// The inner pointer of a `CString` lives only as long as the `CString` it + /// points to. Getting the inner pointer of a *temporary* `CString` allows the `CString` + /// to be dropped at the end of the statement, as it is not being referenced as far as the typesystem + /// is concerned. This means outside of the statement the pointer will point to freed memory, which + /// causes undefined behavior if the pointer is later dereferenced. + pub TEMPORARY_CSTRING_AS_PTR, + Warn, + "detects getting the inner pointer of a temporary `CString`" +} + +declare_lint_pass!(TemporaryCStringAsPtr => [TEMPORARY_CSTRING_AS_PTR]); + +fn in_macro(span: Span) -> bool { + if span.from_expansion() { + !matches!(span.ctxt().outer_expn_data().kind, ExpnKind::Desugaring(..)) + } else { + false + } +} + +fn first_method_call<'tcx>( + expr: &'tcx Expr<'tcx>, +) -> Option<(&'tcx PathSegment<'tcx>, &'tcx [Expr<'tcx>])> { + if let ExprKind::MethodCall(path, _, args, _) = &expr.kind { + if args.iter().any(|e| e.span.from_expansion()) { None } else { Some((path, *args)) } + } else { + None + } +} + +impl<'tcx> LateLintPass<'tcx> for TemporaryCStringAsPtr { + fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) { + if in_macro(expr.span) { + return; + } + + match first_method_call(expr) { + Some((path, args)) if path.ident.name == sym::as_ptr => { + let unwrap_arg = &args[0]; + let as_ptr_span = path.ident.span; + match first_method_call(unwrap_arg) { + Some((path, args)) + if path.ident.name == sym::unwrap || path.ident.name == sym::expect => + { + let source_arg = &args[0]; + lint_cstring_as_ptr(cx, as_ptr_span, source_arg, unwrap_arg); + } + _ => return, + } + } + _ => return, + } + } +} + +fn lint_cstring_as_ptr( + cx: &LateContext<'_>, + as_ptr_span: Span, + source: &rustc_hir::Expr<'_>, + unwrap: &rustc_hir::Expr<'_>, +) { + let source_type = cx.typeck_results().expr_ty(source); + if let ty::Adt(def, substs) = source_type.kind() { + if cx.tcx.is_diagnostic_item(sym::result_type, def.did) { + if let ty::Adt(adt, _) = substs.type_at(0).kind() { + if cx.tcx.is_diagnostic_item(sym::cstring_type, adt.did) { + cx.struct_span_lint(TEMPORARY_CSTRING_AS_PTR, as_ptr_span, |diag| { + let mut diag = diag + .build("getting the inner pointer of a temporary `CString`"); + diag.span_label(as_ptr_span, "this pointer will be invalid"); + diag.span_label( + unwrap.span, + "this `CString` is deallocated at the end of the statement, bind it to a variable to extend its lifetime", + ); + diag.note("pointers do not have a lifetime; when calling `as_ptr` the `CString` will be deallocated at the end of the statement because nothing is referencing it as far as the type system is concerned"); + diag.help("for more information, see https://doc.rust-lang.org/reference/destructors.html"); + diag.emit(); + }); + } + } + } + } +} diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs index b3125f55d4d..f117ce1f805 100644 --- a/compiler/rustc_lint/src/nonstandard_style.rs +++ b/compiler/rustc_lint/src/nonstandard_style.rs @@ -320,7 +320,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase { .with_hi(lit.span.hi() - BytePos(right as u32)), ) }) - .unwrap_or_else(|| lit.span); + .unwrap_or(lit.span); Some(Ident::new(name, sp)) } else { diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs index af14f28ff9f..428cd453dd5 100644 --- a/compiler/rustc_lint/src/types.rs +++ b/compiler/rustc_lint/src/types.rs @@ -18,6 +18,7 @@ use rustc_target::abi::{Integer, LayoutOf, TagEncoding, VariantIdx, Variants}; use rustc_target::spec::abi::Abi as SpecAbi; use std::cmp; +use std::ops::ControlFlow; use tracing::debug; declare_lint! { @@ -145,9 +146,9 @@ fn lint_overflowing_range_endpoint<'tcx>( // We need to preserve the literal's suffix, // as it may determine typing information. let suffix = match lit.node { - LitKind::Int(_, LitIntType::Signed(s)) => s.name_str().to_string(), - LitKind::Int(_, LitIntType::Unsigned(s)) => s.name_str().to_string(), - LitKind::Int(_, LitIntType::Unsuffixed) => "".to_string(), + LitKind::Int(_, LitIntType::Signed(s)) => s.name_str(), + LitKind::Int(_, LitIntType::Unsigned(s)) => s.name_str(), + LitKind::Int(_, LitIntType::Unsuffixed) => "", _ => bug!(), }; let suggestion = format!("{}..={}{}", start, lit_val - 1, suffix); @@ -170,24 +171,25 @@ fn lint_overflowing_range_endpoint<'tcx>( // warnings are consistent between 32- and 64-bit platforms. fn int_ty_range(int_ty: ast::IntTy) -> (i128, i128) { match int_ty { - ast::IntTy::Isize => (i64::MIN as i128, i64::MAX as i128), - ast::IntTy::I8 => (i8::MIN as i64 as i128, i8::MAX as i128), - ast::IntTy::I16 => (i16::MIN as i64 as i128, i16::MAX as i128), - ast::IntTy::I32 => (i32::MIN as i64 as i128, i32::MAX as i128), - ast::IntTy::I64 => (i64::MIN as i128, i64::MAX as i128), - ast::IntTy::I128 => (i128::MIN as i128, i128::MAX), + ast::IntTy::Isize => (i64::MIN.into(), i64::MAX.into()), + ast::IntTy::I8 => (i8::MIN.into(), i8::MAX.into()), + ast::IntTy::I16 => (i16::MIN.into(), i16::MAX.into()), + ast::IntTy::I32 => (i32::MIN.into(), i32::MAX.into()), + ast::IntTy::I64 => (i64::MIN.into(), i64::MAX.into()), + ast::IntTy::I128 => (i128::MIN, i128::MAX), } } fn uint_ty_range(uint_ty: ast::UintTy) -> (u128, u128) { - match uint_ty { - ast::UintTy::Usize => (u64::MIN as u128, u64::MAX as u128), - ast::UintTy::U8 => (u8::MIN as u128, u8::MAX as u128), - ast::UintTy::U16 => (u16::MIN as u128, u16::MAX as u128), - ast::UintTy::U32 => (u32::MIN as u128, u32::MAX as u128), - ast::UintTy::U64 => (u64::MIN as u128, u64::MAX as u128), - ast::UintTy::U128 => (u128::MIN, u128::MAX), - } + let max = match uint_ty { + ast::UintTy::Usize => u64::MAX.into(), + ast::UintTy::U8 => u8::MAX.into(), + ast::UintTy::U16 => u16::MAX.into(), + ast::UintTy::U32 => u32::MAX.into(), + ast::UintTy::U64 => u64::MAX.into(), + ast::UintTy::U128 => u128::MAX, + }; + (0, max) } fn get_bin_hex_repr(cx: &LateContext<'_>, lit: &hir::Lit) -> Option<String> { @@ -438,7 +440,7 @@ fn lint_literal<'tcx>( cx.struct_span_lint(OVERFLOWING_LITERALS, e.span, |lint| { lint.build(&format!("literal out of range for `{}`", t.name_str())) .note(&format!( - "the literal `{}` does not fit into the type `{}` and will be converted to `std::{}::INFINITY`", + "the literal `{}` does not fit into the type `{}` and will be converted to `{}::INFINITY`", cx.sess() .source_map() .span_to_snippet(lit.span) @@ -543,15 +545,15 @@ impl<'tcx> LateLintPass<'tcx> for TypeLimits { } fn is_comparison(binop: hir::BinOp) -> bool { - match binop.node { + matches!( + binop.node, hir::BinOpKind::Eq - | hir::BinOpKind::Lt - | hir::BinOpKind::Le - | hir::BinOpKind::Ne - | hir::BinOpKind::Ge - | hir::BinOpKind::Gt => true, - _ => false, - } + | hir::BinOpKind::Lt + | hir::BinOpKind::Le + | hir::BinOpKind::Ne + | hir::BinOpKind::Ge + | hir::BinOpKind::Gt + ) } } } @@ -1134,11 +1136,11 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { }; impl<'a, 'tcx> ty::fold::TypeVisitor<'tcx> for ProhibitOpaqueTypes<'a, 'tcx> { - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { match ty.kind() { ty::Opaque(..) => { self.ty = Some(ty); - true + ControlFlow::BREAK } // Consider opaque types within projections FFI-safe if they do not normalize // to more opaque types. @@ -1147,7 +1149,11 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { // If `ty` is a opaque type directly then `super_visit_with` won't invoke // this function again. - if ty.has_opaque_types() { self.visit_ty(ty) } else { false } + if ty.has_opaque_types() { + self.visit_ty(ty) + } else { + ControlFlow::CONTINUE + } } _ => ty.super_visit_with(self), } @@ -1232,15 +1238,10 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> { } fn is_internal_abi(&self, abi: SpecAbi) -> bool { - if let SpecAbi::Rust - | SpecAbi::RustCall - | SpecAbi::RustIntrinsic - | SpecAbi::PlatformIntrinsic = abi - { - true - } else { - false - } + matches!( + abi, + SpecAbi::Rust | SpecAbi::RustCall | SpecAbi::RustIntrinsic | SpecAbi::PlatformIntrinsic + ) } } diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs index 4bff7f317cb..4bbc180b226 100644 --- a/compiler/rustc_lint/src/unused.rs +++ b/compiler/rustc_lint/src/unused.rs @@ -250,13 +250,13 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults { has_emitted } ty::Array(ty, len) => match len.try_eval_usize(cx.tcx, cx.param_env) { + // If the array is empty we don't lint, to avoid false positives + Some(0) | None => false, // If the array is definitely non-empty, we can do `#[must_use]` checking. - Some(n) if n != 0 => { + Some(n) => { let descr_pre = &format!("{}array{} of ", descr_pre, plural_suffix,); check_must_use_ty(cx, ty, expr, span, descr_pre, descr_post, n as usize + 1) } - // Otherwise, we don't lint, to avoid false positives. - _ => false, }, ty::Closure(..) => { cx.struct_span_lint(UNUSED_MUST_USE, span, |lint| { @@ -751,13 +751,17 @@ impl UnusedDelimLint for UnusedParens { if !Self::is_expr_delims_necessary(inner, followed_by_block) && value.attrs.is_empty() && !value.span.from_expansion() + && (ctx != UnusedDelimsCtx::LetScrutineeExpr + || !matches!(inner.kind, ast::ExprKind::Binary( + rustc_span::source_map::Spanned { node, .. }, + _, + _, + ) if node.lazy())) { self.emit_unused_delims_expr(cx, value, ctx, left_pos, right_pos) } } ast::ExprKind::Let(_, ref expr) => { - // FIXME(#60336): Properly handle `let true = (false && true)` - // actually needing the parenthesis. self.check_unused_delims_expr( cx, expr, diff --git a/compiler/rustc_lint_defs/Cargo.toml b/compiler/rustc_lint_defs/Cargo.toml new file mode 100644 index 00000000000..7f908088cf5 --- /dev/null +++ b/compiler/rustc_lint_defs/Cargo.toml @@ -0,0 +1,13 @@ +[package] +authors = ["The Rust Project Developers"] +name = "rustc_lint_defs" +version = "0.0.0" +edition = "2018" + +[dependencies] +log = { package = "tracing", version = "0.1" } +rustc_ast = { path = "../rustc_ast" } +rustc_data_structures = { path = "../rustc_data_structures" } +rustc_span = { path = "../rustc_span" } +rustc_serialize = { path = "../rustc_serialize" } +rustc_macros = { path = "../rustc_macros" } diff --git a/compiler/rustc_session/src/lint/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index fef3164de59..048f096aabe 100644 --- a/compiler/rustc_session/src/lint/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -4,7 +4,6 @@ //! compiler code, rather than using their own custom pass. Those //! lints are all available in `rustc_lint::builtin`. -use crate::lint::FutureIncompatibleInfo; use crate::{declare_lint, declare_lint_pass, declare_tool_lint}; use rustc_span::edition::Edition; use rustc_span::symbol::sym; @@ -2647,6 +2646,65 @@ declare_lint! { }; } +declare_lint! { + /// The `function_item_references` lint detects function references that are + /// formatted with [`fmt::Pointer`] or transmuted. + /// + /// [`fmt::Pointer`]: https://doc.rust-lang.org/std/fmt/trait.Pointer.html + /// + /// ### Example + /// + /// ```rust + /// fn foo() { } + /// + /// fn main() { + /// println!("{:p}", &foo); + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Taking a reference to a function may be mistaken as a way to obtain a + /// pointer to that function. This can give unexpected results when + /// formatting the reference as a pointer or transmuting it. This lint is + /// issued when function references are formatted as pointers, passed as + /// arguments bound by [`fmt::Pointer`] or transmuted. + pub FUNCTION_ITEM_REFERENCES, + Warn, + "suggest casting to a function pointer when attempting to take references to function items", +} + +declare_lint! { + /// The `uninhabited_static` lint detects uninhabited statics. + /// + /// ### Example + /// + /// ```rust + /// enum Void {} + /// extern { + /// static EXTERN: Void; + /// } + /// ``` + /// + /// {{produces}} + /// + /// ### Explanation + /// + /// Statics with an uninhabited type can never be initialized, so they are impossible to define. + /// However, this can be side-stepped with an `extern static`, leading to problems later in the + /// compiler which assumes that there are no initialized uninhabited places (such as locals or + /// statics). This was accientally allowed, but is being phased out. + pub UNINHABITED_STATIC, + Warn, + "uninhabited static", + @future_incompatible = FutureIncompatibleInfo { + reference: "issue #74840 <https://github.com/rust-lang/rust/issues/74840>", + edition: None, + }; +} + declare_tool_lint! { pub rustc::INEFFECTIVE_UNSTABLE_TRAIT_IMPL, Deny, @@ -2732,6 +2790,8 @@ declare_lint_pass! { CENUM_IMPL_DROP_CAST, CONST_EVALUATABLE_UNCHECKED, INEFFECTIVE_UNSTABLE_TRAIT_IMPL, + UNINHABITED_STATIC, + FUNCTION_ITEM_REFERENCES, ] } diff --git a/compiler/rustc_session/src/lint.rs b/compiler/rustc_lint_defs/src/lib.rs index 62e021d5e45..25a7bfcabb7 100644 --- a/compiler/rustc_session/src/lint.rs +++ b/compiler/rustc_lint_defs/src/lib.rs @@ -1,12 +1,45 @@ +#[macro_use] +extern crate rustc_macros; + pub use self::Level::*; use rustc_ast::node_id::{NodeId, NodeMap}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey}; -use rustc_errors::{pluralize, Applicability, DiagnosticBuilder}; use rustc_span::edition::Edition; use rustc_span::{sym, symbol::Ident, MultiSpan, Span, Symbol}; pub mod builtin; +#[macro_export] +macro_rules! pluralize { + ($x:expr) => { + if $x != 1 { "s" } else { "" } + }; +} + +/// Indicates the confidence in the correctness of a suggestion. +/// +/// All suggestions are marked with an `Applicability`. Tools use the applicability of a suggestion +/// to determine whether it should be automatically applied or if the user should be consulted +/// before applying the suggestion. +#[derive(Copy, Clone, Debug, PartialEq, Hash, Encodable, Decodable)] +pub enum Applicability { + /// The suggestion is definitely what the user intended. This suggestion should be + /// automatically applied. + MachineApplicable, + + /// The suggestion may be what the user intended, but it is uncertain. The suggestion should + /// result in valid Rust code if it is applied. + MaybeIncorrect, + + /// The suggestion contains placeholders like `(...)` or `{ /* fields */ }`. The suggestion + /// cannot be applied automatically because it will not result in valid Rust code. The user + /// will need to fill in the placeholders. + HasPlaceholders, + + /// The applicability of the suggestion is unknown. + Unspecified, +} + /// Setting for how to handle a lint. #[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)] pub enum Level { @@ -106,6 +139,21 @@ pub struct FutureIncompatibleInfo { /// If this is an edition fixing lint, the edition in which /// this lint becomes obsolete pub edition: Option<Edition>, + /// Information about a future breakage, which will + /// be emitted in JSON messages to be displayed by Cargo + /// for upstream deps + pub future_breakage: Option<FutureBreakage>, +} + +#[derive(Copy, Clone, Debug)] +pub struct FutureBreakage { + pub date: Option<&'static str>, +} + +impl FutureIncompatibleInfo { + pub const fn default_fields_for_macro() -> Self { + FutureIncompatibleInfo { reference: "", edition: None, future_breakage: None } + } } impl Lint { @@ -331,31 +379,34 @@ macro_rules! declare_lint { ); ); ($(#[$attr:meta])* $vis: vis $NAME: ident, $Level: ident, $desc: expr, - $(@future_incompatible = $fi:expr;)? $(@feature_gate = $gate:expr;)? + $(@future_incompatible = FutureIncompatibleInfo { $($field:ident : $val:expr),* $(,)* }; )? $($v:ident),*) => ( $(#[$attr])* - $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint { + $vis static $NAME: &$crate::Lint = &$crate::Lint { name: stringify!($NAME), - default_level: $crate::lint::$Level, + default_level: $crate::$Level, desc: $desc, edition_lint_opts: None, is_plugin: false, $($v: true,)* - $(future_incompatible: Some($fi),)* $(feature_gate: Some($gate),)* - ..$crate::lint::Lint::default_fields_for_macro() + $(future_incompatible: Some($crate::FutureIncompatibleInfo { + $($field: $val,)* + ..$crate::FutureIncompatibleInfo::default_fields_for_macro() + }),)* + ..$crate::Lint::default_fields_for_macro() }; ); ($(#[$attr:meta])* $vis: vis $NAME: ident, $Level: ident, $desc: expr, $lint_edition: expr => $edition_level: ident ) => ( $(#[$attr])* - $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint { + $vis static $NAME: &$crate::Lint = &$crate::Lint { name: stringify!($NAME), - default_level: $crate::lint::$Level, + default_level: $crate::$Level, desc: $desc, - edition_lint_opts: Some(($lint_edition, $crate::lint::Level::$edition_level)), + edition_lint_opts: Some(($lint_edition, $crate::Level::$edition_level)), report_in_external_macro: false, is_plugin: false, }; @@ -380,9 +431,9 @@ macro_rules! declare_tool_lint { $external:expr ) => ( $(#[$attr])* - $vis static $NAME: &$crate::lint::Lint = &$crate::lint::Lint { + $vis static $NAME: &$crate::Lint = &$crate::Lint { name: &concat!(stringify!($tool), "::", stringify!($NAME)), - default_level: $crate::lint::$Level, + default_level: $crate::$Level, desc: $desc, edition_lint_opts: None, report_in_external_macro: $external, @@ -413,11 +464,11 @@ pub trait LintPass { #[macro_export] macro_rules! impl_lint_pass { ($ty:ty => [$($lint:expr),* $(,)?]) => { - impl $crate::lint::LintPass for $ty { + impl $crate::LintPass for $ty { fn name(&self) -> &'static str { stringify!($ty) } } impl $ty { - pub fn get_lints() -> $crate::lint::LintArray { $crate::lint_array!($($lint),*) } + pub fn get_lints() -> $crate::LintArray { $crate::lint_array!($($lint),*) } } }; } @@ -431,45 +482,3 @@ macro_rules! declare_lint_pass { $crate::impl_lint_pass!($name => [$($lint),*]); }; } - -pub fn add_elided_lifetime_in_path_suggestion( - sess: &crate::Session, - db: &mut DiagnosticBuilder<'_>, - n: usize, - path_span: Span, - incl_angl_brckt: bool, - insertion_span: Span, - anon_lts: String, -) { - let (replace_span, suggestion) = if incl_angl_brckt { - (insertion_span, anon_lts) - } else { - // When possible, prefer a suggestion that replaces the whole - // `Path<T>` expression with `Path<'_, T>`, rather than inserting `'_, ` - // at a point (which makes for an ugly/confusing label) - if let Ok(snippet) = sess.source_map().span_to_snippet(path_span) { - // But our spans can get out of whack due to macros; if the place we think - // we want to insert `'_` isn't even within the path expression's span, we - // should bail out of making any suggestion rather than panicking on a - // subtract-with-overflow or string-slice-out-out-bounds (!) - // FIXME: can we do better? - if insertion_span.lo().0 < path_span.lo().0 { - return; - } - let insertion_index = (insertion_span.lo().0 - path_span.lo().0) as usize; - if insertion_index > snippet.len() { - return; - } - let (before, after) = snippet.split_at(insertion_index); - (path_span, format!("{}{}{}", before, anon_lts, after)) - } else { - (insertion_span, anon_lts) - } - }; - db.span_suggestion( - replace_span, - &format!("indicate the anonymous lifetime{}", pluralize!(n)), - suggestion, - Applicability::MachineApplicable, - ); -} diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 9f8ea7f43d8..40a10434248 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -733,7 +733,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( const char *LinkageName, size_t LinkageNameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, - LLVMRustDISPFlags SPFlags, LLVMValueRef Fn, LLVMMetadataRef TParam, + LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, LLVMMetadataRef Decl) { DITemplateParameterArray TParams = DITemplateParameterArray(unwrap<MDTuple>(TParam)); @@ -750,7 +750,8 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( unwrapDI<DIFile>(File), LineNo, unwrapDI<DISubroutineType>(Ty), ScopeLine, llvmFlags, llvmSPFlags, TParams, unwrapDIPtr<DISubprogram>(Decl)); - unwrap<Function>(Fn)->setSubprogram(Sub); + if (MaybeFn) + unwrap<Function>(MaybeFn)->setSubprogram(Sub); return wrap(Sub); } @@ -765,7 +766,7 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { return wrap(Builder->createTypedef( unwrap<DIType>(Type), StringRef(Name, NameLen), unwrap<DIFile>(File), - LineNo, unwrap<DIScope>(Scope))); + LineNo, unwrapDIPtr<DIScope>(Scope))); } extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( @@ -930,12 +931,12 @@ LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, - int64_t *AddrOps, unsigned AddrOpsCount, LLVMValueRef DL, + int64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, LLVMBasicBlockRef InsertAtEnd) { return wrap(Builder->insertDeclare( unwrap(V), unwrap<DILocalVariable>(VarInfo), Builder->createExpression(llvm::ArrayRef<int64_t>(AddrOps, AddrOpsCount)), - DebugLoc(cast<MDNode>(unwrap<MetadataAsValue>(DL)->getMetadata())), + DebugLoc(cast<MDNode>(unwrap(DL))), unwrap(InsertAtEnd))); } @@ -1002,7 +1003,7 @@ LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, DINodeArray(unwrap<MDTuple>(Params))); } -extern "C" LLVMValueRef +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateDebugLocation(LLVMContextRef ContextRef, unsigned Line, unsigned Column, LLVMMetadataRef Scope, LLVMMetadataRef InlinedAt) { @@ -1011,7 +1012,7 @@ LLVMRustDIBuilderCreateDebugLocation(LLVMContextRef ContextRef, unsigned Line, DebugLoc debug_loc = DebugLoc::get(Line, Column, unwrapDIPtr<MDNode>(Scope), unwrapDIPtr<MDNode>(InlinedAt)); - return wrap(MetadataAsValue::get(Context, debug_loc.getAsMDNode())); + return wrap(debug_loc.getAsMDNode()); } extern "C" int64_t LLVMRustDIBuilderCreateOpDeref() { diff --git a/compiler/rustc_macros/src/lift.rs b/compiler/rustc_macros/src/lift.rs index 4bf4ce00a4d..ad7ac740417 100644 --- a/compiler/rustc_macros/src/lift.rs +++ b/compiler/rustc_macros/src/lift.rs @@ -3,6 +3,7 @@ use syn::{self, parse_quote}; pub fn lift_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream { s.add_bounds(synstructure::AddBounds::Generics); + s.bind_with(|_| synstructure::BindStyle::Move); let tcx: syn::Lifetime = parse_quote!('tcx); let newtcx: syn::GenericParam = parse_quote!('__lifted); @@ -43,8 +44,8 @@ pub fn lift_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStre quote! { type Lifted = #lifted; - fn lift_to_tcx(&self, __tcx: ::rustc_middle::ty::TyCtxt<'__lifted>) -> Option<#lifted> { - Some(match *self { #body }) + fn lift_to_tcx(self, __tcx: ::rustc_middle::ty::TyCtxt<'__lifted>) -> Option<#lifted> { + Some(match self { #body }) } }, ) diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs index e7c054653ac..fd859196369 100644 --- a/compiler/rustc_macros/src/query.rs +++ b/compiler/rustc_macros/src/query.rs @@ -190,7 +190,11 @@ impl<T: Parse> Parse for List<T> { } /// A named group containing queries. +/// +/// For now, the name is not used any more, but the capability remains interesting for future +/// developments of the query system. struct Group { + #[allow(unused)] name: Ident, queries: List<Query>, } @@ -417,12 +421,9 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { let mut query_stream = quote! {}; let mut query_description_stream = quote! {}; let mut dep_node_def_stream = quote! {}; - let mut dep_node_force_stream = quote! {}; - let mut try_load_from_on_disk_cache_stream = quote! {}; let mut cached_queries = quote! {}; for group in groups.0 { - let mut group_stream = quote! {}; for mut query in group.queries.0 { let modifiers = process_modifiers(&mut query); let name = &query.name; @@ -437,22 +438,6 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { cached_queries.extend(quote! { #name, }); - - try_load_from_on_disk_cache_stream.extend(quote! { - ::rustc_middle::dep_graph::DepKind::#name => { - if <#arg as DepNodeParams<TyCtxt<'_>>>::can_reconstruct_query_key() { - debug_assert!($tcx.dep_graph - .node_color($dep_node) - .map(|c| c.is_green()) - .unwrap_or(false)); - - let key = <#arg as DepNodeParams<TyCtxt<'_>>>::recover($tcx, $dep_node).unwrap(); - if queries::#name::cache_on_disk($tcx, &key, None) { - let _ = $tcx.#name(key); - } - } - } - }); } let mut attributes = Vec::new(); @@ -485,9 +470,9 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { let attribute_stream = quote! {#(#attributes),*}; let doc_comments = query.doc_comments.iter(); // Add the query to the group - group_stream.extend(quote! { + query_stream.extend(quote! { #(#doc_comments)* - [#attribute_stream] fn #name: #name(#arg) #result, + [#attribute_stream] fn #name(#arg) #result, }); // Create a dep node for the query @@ -495,37 +480,10 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { [#attribute_stream] #name(#arg), }); - // Add a match arm to force the query given the dep node - dep_node_force_stream.extend(quote! { - ::rustc_middle::dep_graph::DepKind::#name => { - if <#arg as DepNodeParams<TyCtxt<'_>>>::can_reconstruct_query_key() { - if let Some(key) = <#arg as DepNodeParams<TyCtxt<'_>>>::recover($tcx, $dep_node) { - force_query::<crate::ty::query::queries::#name<'_>, _>( - $tcx, - key, - DUMMY_SP, - *$dep_node - ); - return true; - } - } - } - }); - add_query_description_impl(&query, modifiers, &mut query_description_stream); } - let name = &group.name; - query_stream.extend(quote! { - #name { #group_stream }, - }); } - dep_node_force_stream.extend(quote! { - ::rustc_middle::dep_graph::DepKind::Null => { - bug!("Cannot force dep node: {:?}", $dep_node) - } - }); - TokenStream::from(quote! { macro_rules! rustc_query_append { ([$($macro:tt)*][$($other:tt)*]) => { @@ -546,15 +504,6 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { ); } } - macro_rules! rustc_dep_node_force { - ([$dep_node:expr, $tcx:expr] $($other:tt)*) => { - match $dep_node.kind { - $($other)* - - #dep_node_force_stream - } - } - } macro_rules! rustc_cached_queries { ($($macro:tt)*) => { $($macro)*(#cached_queries); @@ -562,14 +511,5 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { } #query_description_stream - - macro_rules! rustc_dep_node_try_load_from_on_disk_cache { - ($dep_node:expr, $tcx:expr) => { - match $dep_node.kind { - #try_load_from_on_disk_cache_stream - _ => (), - } - } - } }) } diff --git a/compiler/rustc_macros/src/type_foldable.rs b/compiler/rustc_macros/src/type_foldable.rs index 6931e6552ad..b16e22e8d77 100644 --- a/compiler/rustc_macros/src/type_foldable.rs +++ b/compiler/rustc_macros/src/type_foldable.rs @@ -15,8 +15,12 @@ pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2:: } }) }); - let body_visit = s.fold(false, |acc, bind| { - quote! { #acc || ::rustc_middle::ty::fold::TypeFoldable::visit_with(#bind, __folder) } + + let body_visit = s.fold(quote!(), |acc, bind| { + quote! { + #acc + ::rustc_middle::ty::fold::TypeFoldable::visit_with(#bind, __folder)?; + } }); s.bound_impl( @@ -32,8 +36,9 @@ pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2:: fn super_visit_with<__F: ::rustc_middle::ty::fold::TypeVisitor<'tcx>>( &self, __folder: &mut __F - ) -> bool { + ) -> ::std::ops::ControlFlow<()> { match *self { #body_visit } + ::std::ops::ControlFlow::CONTINUE } }, ) diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index 7562da6d782..33cbf0fb234 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -752,10 +752,7 @@ impl<'a> CrateLoader<'a> { // At this point we've determined that we need an allocator. Let's see // if our compilation session actually needs an allocator based on what // we're emitting. - let all_rlib = self.sess.crate_types().iter().all(|ct| match *ct { - CrateType::Rlib => true, - _ => false, - }); + let all_rlib = self.sess.crate_types().iter().all(|ct| matches!(*ct, CrateType::Rlib)); if all_rlib { return; } diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index f225f8acc89..d16985b9c2b 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -633,11 +633,9 @@ impl<'a> CrateLocator<'a> { } } - if self.exact_paths.is_empty() { - if self.crate_name != root.name() { - info!("Rejecting via crate name"); - return None; - } + if self.exact_paths.is_empty() && self.crate_name != root.name() { + info!("Rejecting via crate name"); + return None; } if root.triple() != &self.triple { diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 05b8dad3097..b5fb850e92e 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -8,6 +8,7 @@ use rustc_ast as ast; use rustc_ast::expand::allocator::AllocatorKind; use rustc_data_structures::svh::Svh; use rustc_hir as hir; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_middle::hir::exports::Export; @@ -219,10 +220,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, missing_lang_items => { cdata.get_missing_lang_items(tcx) } missing_extern_crate_item => { - let r = match *cdata.extern_crate.borrow() { - Some(extern_crate) if !extern_crate.is_direct() => true, - _ => false, - }; + let r = matches!(*cdata.extern_crate.borrow(), Some(extern_crate) if !extern_crate.is_direct()); r } @@ -253,9 +251,11 @@ pub fn provide(providers: &mut Providers) { } _ => false, }, - is_statically_included_foreign_item: |tcx, id| match tcx.native_library_kind(id) { - Some(NativeLibKind::StaticBundle | NativeLibKind::StaticNoBundle) => true, - _ => false, + is_statically_included_foreign_item: |tcx, id| { + matches!( + tcx.native_library_kind(id), + Some(NativeLibKind::StaticBundle | NativeLibKind::StaticNoBundle) + ) }, native_library_kind: |tcx, id| { tcx.native_libraries(id.krate) @@ -487,6 +487,10 @@ impl CrateStore for CStore { self.get_crate_data(def.krate).def_key(def.index) } + fn def_kind(&self, def: DefId) -> DefKind { + self.get_crate_data(def.krate).def_kind(def.index) + } + fn def_path(&self, def: DefId) -> DefPath { self.get_crate_data(def.krate).def_path(def.index) } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index f0911928e81..2a81737e168 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -28,7 +28,6 @@ use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt}; use rustc_serialize::{opaque, Encodable, Encoder}; use rustc_session::config::CrateType; use rustc_span::hygiene::{ExpnDataEncodeMode, HygieneEncodeContext}; -use rustc_span::source_map::Spanned; use rustc_span::symbol::{sym, Ident, Symbol}; use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SyntaxContext}; use rustc_target::abi::VariantIdx; @@ -436,8 +435,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { fn encode_info_for_items(&mut self) { let krate = self.tcx.hir().krate(); - let vis = Spanned { span: rustc_span::DUMMY_SP, node: hir::VisibilityKind::Public }; - self.encode_info_for_mod(hir::CRATE_HIR_ID, &krate.item.module, &krate.item.attrs, &vis); + self.encode_info_for_mod(hir::CRATE_HIR_ID, &krate.item.module, &krate.item.attrs); // Proc-macro crates only export proc-macro items, which are looked // up using `proc_macro_data` @@ -739,12 +737,8 @@ impl EncodeContext<'a, 'tcx> { is_non_exhaustive: variant.is_field_list_non_exhaustive(), }; - let enum_id = tcx.hir().local_def_id_to_hir_id(def.did.expect_local()); - let enum_vis = &tcx.hir().expect_item(enum_id).vis; - record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data))); - record!(self.tables.visibility[def_id] <- - ty::Visibility::from_hir(enum_vis, enum_id, self.tcx)); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); record!(self.tables.attributes[def_id] <- &self.tcx.get_attrs(def_id)[..]); record!(self.tables.expn_that_defined[def_id] <- self.tcx.expansion_that_defined(def_id)); @@ -785,17 +779,8 @@ impl EncodeContext<'a, 'tcx> { is_non_exhaustive: variant.is_field_list_non_exhaustive(), }; - // Variant constructors have the same visibility as the parent enums, unless marked as - // non-exhaustive, in which case they are lowered to `pub(crate)`. - let enum_id = tcx.hir().local_def_id_to_hir_id(def.did.expect_local()); - let enum_vis = &tcx.hir().expect_item(enum_id).vis; - let mut ctor_vis = ty::Visibility::from_hir(enum_vis, enum_id, tcx); - if variant.is_field_list_non_exhaustive() && ctor_vis == ty::Visibility::Public { - ctor_vis = ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)); - } - record!(self.tables.kind[def_id] <- EntryKind::Variant(self.lazy(data))); - record!(self.tables.visibility[def_id] <- ctor_vis); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); self.encode_stability(def_id); self.encode_deprecation(def_id); @@ -811,13 +796,7 @@ impl EncodeContext<'a, 'tcx> { self.encode_promoted_mir(def_id.expect_local()); } - fn encode_info_for_mod( - &mut self, - id: hir::HirId, - md: &hir::Mod<'_>, - attrs: &[ast::Attribute], - vis: &hir::Visibility<'_>, - ) { + fn encode_info_for_mod(&mut self, id: hir::HirId, md: &hir::Mod<'_>, attrs: &[ast::Attribute]) { let tcx = self.tcx; let local_def_id = tcx.hir().local_def_id(id); let def_id = local_def_id.to_def_id(); @@ -850,7 +829,7 @@ impl EncodeContext<'a, 'tcx> { }; record!(self.tables.kind[def_id] <- EntryKind::Mod(self.lazy(data))); - record!(self.tables.visibility[def_id] <- ty::Visibility::from_hir(vis, id, self.tcx)); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); record!(self.tables.attributes[def_id] <- attrs); if self.is_proc_macro { @@ -881,7 +860,7 @@ impl EncodeContext<'a, 'tcx> { let variant_data = tcx.hir().expect_variant_data(variant_id); record!(self.tables.kind[def_id] <- EntryKind::Field); - record!(self.tables.visibility[def_id] <- field.vis); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); record!(self.tables.attributes[def_id] <- variant_data.fields()[field_index].attrs); record!(self.tables.expn_that_defined[def_id] <- self.tcx.expansion_that_defined(def_id)); @@ -906,25 +885,8 @@ impl EncodeContext<'a, 'tcx> { is_non_exhaustive: variant.is_field_list_non_exhaustive(), }; - let struct_id = tcx.hir().local_def_id_to_hir_id(adt_def.did.expect_local()); - let struct_vis = &tcx.hir().expect_item(struct_id).vis; - let mut ctor_vis = ty::Visibility::from_hir(struct_vis, struct_id, tcx); - for field in &variant.fields { - if ctor_vis.is_at_least(field.vis, tcx) { - ctor_vis = field.vis; - } - } - - // If the structure is marked as non_exhaustive then lower the visibility - // to within the crate. - if adt_def.non_enum_variant().is_field_list_non_exhaustive() - && ctor_vis == ty::Visibility::Public - { - ctor_vis = ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)); - } - record!(self.tables.kind[def_id] <- EntryKind::Struct(self.lazy(data), adt_def.repr)); - record!(self.tables.visibility[def_id] <- ctor_vis); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); record!(self.tables.expn_that_defined[def_id] <- self.tcx.expansion_that_defined(def_id)); self.encode_stability(def_id); @@ -1030,7 +992,7 @@ impl EncodeContext<'a, 'tcx> { EntryKind::AssocType(container) } }); - record!(self.tables.visibility[def_id] <- trait_item.vis); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- ast_item.span); record!(self.tables.attributes[def_id] <- ast_item.attrs); self.encode_ident_span(def_id, ast_item.ident); @@ -1112,7 +1074,7 @@ impl EncodeContext<'a, 'tcx> { } ty::AssocKind::Type => EntryKind::AssocType(container) }); - record!(self.tables.visibility[def_id] <- impl_item.vis); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- ast_item.span); record!(self.tables.attributes[def_id] <- ast_item.attrs); self.encode_ident_span(def_id, impl_item.ident); @@ -1261,7 +1223,7 @@ impl EncodeContext<'a, 'tcx> { EntryKind::Fn(self.lazy(data)) } hir::ItemKind::Mod(ref m) => { - return self.encode_info_for_mod(item.hir_id, m, &item.attrs, &item.vis); + return self.encode_info_for_mod(item.hir_id, m, &item.attrs); } hir::ItemKind::ForeignMod(_) => EntryKind::ForeignMod, hir::ItemKind::GlobalAsm(..) => EntryKind::GlobalAsm, @@ -1352,8 +1314,7 @@ impl EncodeContext<'a, 'tcx> { hir::ItemKind::ExternCrate(_) | hir::ItemKind::Use(..) => bug!("cannot encode info for item {:?}", item), }); - record!(self.tables.visibility[def_id] <- - ty::Visibility::from_hir(&item.vis, item.hir_id, tcx)); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); record!(self.tables.attributes[def_id] <- item.attrs); record!(self.tables.expn_that_defined[def_id] <- self.tcx.expansion_that_defined(def_id)); @@ -1470,7 +1431,7 @@ impl EncodeContext<'a, 'tcx> { fn encode_info_for_macro_def(&mut self, macro_def: &hir::MacroDef<'_>) { let def_id = self.tcx.hir().local_def_id(macro_def.hir_id).to_def_id(); record!(self.tables.kind[def_id] <- EntryKind::MacroDef(self.lazy(macro_def.ast.clone()))); - record!(self.tables.visibility[def_id] <- ty::Visibility::Public); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- macro_def.span); record!(self.tables.attributes[def_id] <- macro_def.attrs); self.encode_ident_span(def_id, macro_def.ident); @@ -1480,7 +1441,6 @@ impl EncodeContext<'a, 'tcx> { fn encode_info_for_generic_param(&mut self, def_id: DefId, kind: EntryKind, encode_type: bool) { record!(self.tables.kind[def_id] <- kind); - record!(self.tables.visibility[def_id] <- ty::Visibility::Public); record!(self.tables.span[def_id] <- self.tcx.def_span(def_id)); if encode_type { self.encode_item_type(def_id); @@ -1505,7 +1465,6 @@ impl EncodeContext<'a, 'tcx> { _ => bug!("closure that is neither generator nor closure"), }); - record!(self.tables.visibility[def_id.to_def_id()] <- ty::Visibility::Public); record!(self.tables.span[def_id.to_def_id()] <- self.tcx.def_span(def_id)); record!(self.tables.attributes[def_id.to_def_id()] <- &self.tcx.get_attrs(def_id.to_def_id())[..]); self.encode_item_type(def_id.to_def_id()); @@ -1525,7 +1484,6 @@ impl EncodeContext<'a, 'tcx> { let qualifs = self.tcx.mir_const_qualif(def_id); record!(self.tables.kind[def_id.to_def_id()] <- EntryKind::AnonConst(qualifs, const_data)); - record!(self.tables.visibility[def_id.to_def_id()] <- ty::Visibility::Public); record!(self.tables.span[def_id.to_def_id()] <- self.tcx.def_span(def_id)); self.encode_item_type(def_id.to_def_id()); self.encode_generics(def_id.to_def_id()); @@ -1762,8 +1720,7 @@ impl EncodeContext<'a, 'tcx> { hir::ForeignItemKind::Static(_, hir::Mutability::Not) => EntryKind::ForeignImmStatic, hir::ForeignItemKind::Type => EntryKind::ForeignType, }); - record!(self.tables.visibility[def_id] <- - ty::Visibility::from_hir(&nitem.vis, nitem.hir_id, self.tcx)); + record!(self.tables.visibility[def_id] <- self.tcx.visibility(def_id)); record!(self.tables.span[def_id] <- nitem.span); record!(self.tables.attributes[def_id] <- nitem.attrs); self.encode_ident_span(def_id, nitem.ident); @@ -2085,6 +2042,10 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { encoder.emit_raw_bytes(&[0, 0, 0, 0]); let source_map_files = tcx.sess.source_map().files(); + let source_file_cache = (source_map_files[0].clone(), 0); + let required_source_files = Some(GrowableBitSet::with_capacity(source_map_files.len())); + drop(source_map_files); + let hygiene_ctxt = HygieneEncodeContext::default(); let mut ecx = EncodeContext { @@ -2095,13 +2056,12 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { lazy_state: LazyState::NoNode, type_shorthands: Default::default(), predicate_shorthands: Default::default(), - source_file_cache: (source_map_files[0].clone(), 0), + source_file_cache, interpret_allocs: Default::default(), - required_source_files: Some(GrowableBitSet::with_capacity(source_map_files.len())), + required_source_files, is_proc_macro: tcx.sess.crate_types().contains(&CrateType::ProcMacro), hygiene_ctxt: &hygiene_ctxt, }; - drop(source_map_files); // Encode the rustc version string in a predictable location. rustc_version().encode(&mut ecx).unwrap(); diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index bdbc9ab52bf..66532ea02f3 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -28,5 +28,5 @@ rustc_ast = { path = "../rustc_ast" } rustc_span = { path = "../rustc_span" } chalk-ir = "0.32.0" smallvec = { version = "1.0", features = ["union", "may_dangle"] } -measureme = "0.7.1" +measureme = "9.0.0" rustc_session = { path = "../rustc_session" } diff --git a/compiler/rustc_middle/src/hir/map/collector.rs b/compiler/rustc_middle/src/hir/map/collector.rs index d6869ab8875..516c9b6752b 100644 --- a/compiler/rustc_middle/src/hir/map/collector.rs +++ b/compiler/rustc_middle/src/hir/map/collector.rs @@ -360,8 +360,26 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { } fn visit_generic_param(&mut self, param: &'hir GenericParam<'hir>) { - self.insert(param.span, param.hir_id, Node::GenericParam(param)); - intravisit::walk_generic_param(self, param); + if let hir::GenericParamKind::Type { + synthetic: Some(hir::SyntheticTyParamKind::ImplTrait), + .. + } = param.kind + { + debug_assert_eq!( + param.hir_id.owner, + self.definitions.opt_hir_id_to_local_def_id(param.hir_id).unwrap() + ); + self.with_dep_node_owner(param.hir_id.owner, param, |this, hash| { + this.insert_with_hash(param.span, param.hir_id, Node::GenericParam(param), hash); + + this.with_parent(param.hir_id, |this| { + intravisit::walk_generic_param(this, param); + }); + }); + } else { + self.insert(param.span, param.hir_id, Node::GenericParam(param)); + intravisit::walk_generic_param(self, param); + } } fn visit_trait_item(&mut self, ti: &'hir TraitItem<'hir>) { diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs index 57f03c2a5cf..106fa8c78fa 100644 --- a/compiler/rustc_middle/src/hir/map/mod.rs +++ b/compiler/rustc_middle/src/hir/map/mod.rs @@ -816,7 +816,7 @@ impl<'hir> Map<'hir> { Some(Node::Variant(ref v)) => Some(&v.attrs[..]), Some(Node::Field(ref f)) => Some(&f.attrs[..]), Some(Node::Expr(ref e)) => Some(&*e.attrs), - Some(Node::Stmt(ref s)) => Some(s.kind.attrs()), + Some(Node::Stmt(ref s)) => Some(s.kind.attrs(|id| self.item(id.id))), Some(Node::Arm(ref a)) => Some(&*a.attrs), Some(Node::GenericParam(param)) => Some(¶m.attrs[..]), // Unit/tuple structs/variants take the attributes straight from diff --git a/compiler/rustc_middle/src/ich/impls_syntax.rs b/compiler/rustc_middle/src/ich/impls_syntax.rs index 7aba4fc64a9..cab2ca2919f 100644 --- a/compiler/rustc_middle/src/ich/impls_syntax.rs +++ b/compiler/rustc_middle/src/ich/impls_syntax.rs @@ -40,11 +40,12 @@ impl<'ctx> rustc_ast::HashStableContext for StableHashingContext<'ctx> { debug_assert!(!attr.ident().map_or(false, |ident| self.is_ignored_attr(ident.name))); debug_assert!(!attr.is_doc_comment()); - let ast::Attribute { kind, id: _, style, span } = attr; + let ast::Attribute { kind, id: _, style, span, tokens } = attr; if let ast::AttrKind::Normal(item) = kind { item.hash_stable(self, hasher); style.hash_stable(self, hasher); span.hash_stable(self, hasher); + tokens.as_ref().expect_none("Tokens should have been removed during lowering!"); } else { unreachable!(); } diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs index 4d884dde393..16e9aafb25a 100644 --- a/compiler/rustc_middle/src/infer/unify_key.rs +++ b/compiler/rustc_middle/src/infer/unify_key.rs @@ -175,19 +175,15 @@ impl<'tcx> UnifyKey for ty::ConstVid<'tcx> { impl<'tcx> UnifyValue for ConstVarValue<'tcx> { type Error = (&'tcx ty::Const<'tcx>, &'tcx ty::Const<'tcx>); - fn unify_values(value1: &Self, value2: &Self) -> Result<Self, Self::Error> { - let (val, span) = match (value1.val, value2.val) { + fn unify_values(&value1: &Self, &value2: &Self) -> Result<Self, Self::Error> { + Ok(match (value1.val, value2.val) { (ConstVariableValue::Known { .. }, ConstVariableValue::Known { .. }) => { bug!("equating two const variables, both of which have known values") } // If one side is known, prefer that one. - (ConstVariableValue::Known { .. }, ConstVariableValue::Unknown { .. }) => { - (value1.val, value1.origin.span) - } - (ConstVariableValue::Unknown { .. }, ConstVariableValue::Known { .. }) => { - (value2.val, value2.origin.span) - } + (ConstVariableValue::Known { .. }, ConstVariableValue::Unknown { .. }) => value1, + (ConstVariableValue::Unknown { .. }, ConstVariableValue::Known { .. }) => value2, // If both sides are *unknown*, it hardly matters, does it? ( @@ -200,16 +196,11 @@ impl<'tcx> UnifyValue for ConstVarValue<'tcx> { // universe is the minimum of the two universes, because that is // the one which contains the fewest names in scope. let universe = cmp::min(universe1, universe2); - (ConstVariableValue::Unknown { universe }, value1.origin.span) + ConstVarValue { + val: ConstVariableValue::Unknown { universe }, + origin: value1.origin, + } } - }; - - Ok(ConstVarValue { - origin: ConstVariableOrigin { - kind: ConstVariableOriginKind::ConstInference, - span: span, - }, - val, }) } } diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index fa885ce2e7c..4b915fe020f 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -47,6 +47,9 @@ #![feature(associated_type_bounds)] #![feature(rustc_attrs)] #![feature(int_error_matching)] +#![feature(half_open_range_patterns)] +#![feature(exclusive_range_pattern)] +#![feature(control_flow_enum)] #![recursion_limit = "512"] #[macro_use] diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs index 91e1d6e0b0b..6e67f0d828c 100644 --- a/compiler/rustc_middle/src/lint.rs +++ b/compiler/rustc_middle/src/lint.rs @@ -225,9 +225,24 @@ pub fn struct_lint_level<'s, 'd>( span: Option<MultiSpan>, decorate: Box<dyn for<'b> FnOnce(LintDiagnosticBuilder<'b>) + 'd>, ) { + // Check for future incompatibility lints and issue a stronger warning. + let lint_id = LintId::of(lint); + let future_incompatible = lint.future_incompatible; + + let has_future_breakage = + future_incompatible.map_or(false, |incompat| incompat.future_breakage.is_some()); + let mut err = match (level, span) { - (Level::Allow, _) => { - return; + (Level::Allow, span) => { + if has_future_breakage { + if let Some(span) = span { + sess.struct_span_allow(span, "") + } else { + sess.struct_allow("") + } + } else { + return; + } } (Level::Warn, Some(span)) => sess.struct_span_warn(span, ""), (Level::Warn, None) => sess.struct_warn(""), @@ -235,10 +250,6 @@ pub fn struct_lint_level<'s, 'd>( (Level::Deny | Level::Forbid, None) => sess.struct_err(""), }; - // Check for future incompatibility lints and issue a stronger warning. - let lint_id = LintId::of(lint); - let future_incompatible = lint.future_incompatible; - // If this code originates in a foreign macro, aka something that this crate // did not itself author, then it's likely that there's nothing this crate // can do about it. We probably want to skip the lint entirely. @@ -321,7 +332,7 @@ pub fn struct_lint_level<'s, 'd>( } } - err.code(DiagnosticId::Lint(name)); + err.code(DiagnosticId::Lint { name, has_future_breakage }); if let Some(future_incompatible) = future_incompatible { const STANDARD_MESSAGE: &str = "this was previously accepted by the compiler but is being phased out; \ diff --git a/compiler/rustc_middle/src/macros.rs b/compiler/rustc_middle/src/macros.rs index a5482b7bdcf..921086366be 100644 --- a/compiler/rustc_middle/src/macros.rs +++ b/compiler/rustc_middle/src/macros.rs @@ -29,8 +29,8 @@ macro_rules! CloneLiftImpls { $( impl<$tcx> $crate::ty::Lift<$tcx> for $ty { type Lifted = Self; - fn lift_to_tcx(&self, _: $crate::ty::TyCtxt<$tcx>) -> Option<Self> { - Some(Clone::clone(self)) + fn lift_to_tcx(self, _: $crate::ty::TyCtxt<$tcx>) -> Option<Self> { + Some(self) } } )+ @@ -62,9 +62,9 @@ macro_rules! CloneTypeFoldableImpls { fn super_visit_with<F: $crate::ty::fold::TypeVisitor<$tcx>>( &self, _: &mut F) - -> bool + -> ::std::ops::ControlFlow<()> { - false + ::std::ops::ControlFlow::CONTINUE } } )+ @@ -105,7 +105,7 @@ macro_rules! EnumTypeFoldableImpl { fn super_visit_with<V: $crate::ty::fold::TypeVisitor<$tcx>>( &self, visitor: &mut V, - ) -> bool { + ) -> ::std::ops::ControlFlow<()> { EnumTypeFoldableImpl!(@VisitVariants(self, visitor) input($($variants)*) output()) } } @@ -179,9 +179,10 @@ macro_rules! EnumTypeFoldableImpl { input($($input)*) output( $variant ( $($variant_arg),* ) => { - false $(|| $crate::ty::fold::TypeFoldable::visit_with( + $($crate::ty::fold::TypeFoldable::visit_with( $variant_arg, $visitor - ))* + )?;)* + ::std::ops::ControlFlow::CONTINUE } $($output)* ) @@ -196,9 +197,10 @@ macro_rules! EnumTypeFoldableImpl { input($($input)*) output( $variant { $($variant_arg),* } => { - false $(|| $crate::ty::fold::TypeFoldable::visit_with( + $($crate::ty::fold::TypeFoldable::visit_with( $variant_arg, $visitor - ))* + )?;)* + ::std::ops::ControlFlow::CONTINUE } $($output)* ) @@ -212,7 +214,7 @@ macro_rules! EnumTypeFoldableImpl { @VisitVariants($this, $visitor) input($($input)*) output( - $variant => { false } + $variant => { ::std::ops::ControlFlow::CONTINUE } $($output)* ) ) diff --git a/compiler/rustc_middle/src/middle/cstore.rs b/compiler/rustc_middle/src/middle/cstore.rs index f3d7c8506ab..ae9e4d364d3 100644 --- a/compiler/rustc_middle/src/middle/cstore.rs +++ b/compiler/rustc_middle/src/middle/cstore.rs @@ -8,6 +8,7 @@ use rustc_ast as ast; use rustc_ast::expand::allocator::AllocatorKind; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{self, MetadataRef}; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_macros::HashStable; @@ -185,6 +186,7 @@ pub trait CrateStore { // resolve fn def_key(&self, def: DefId) -> DefKey; + fn def_kind(&self, def: DefId) -> DefKind; fn def_path(&self, def: DefId) -> DefPath; fn def_path_hash(&self, def: DefId) -> DefPathHash; fn all_def_path_hashes_and_def_ids(&self, cnum: CrateNum) -> Vec<(DefPathHash, DefId)>; diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs index 38cb3c1701f..d060549ca81 100644 --- a/compiler/rustc_middle/src/middle/region.rs +++ b/compiler/rustc_middle/src/middle/region.rs @@ -292,20 +292,20 @@ pub struct ScopeTree { /// /// Then: /// - /// 1. From the ordering guarantee of HIR visitors (see - /// `rustc_hir::intravisit`), `D` does not dominate `U`. + /// 1. From the ordering guarantee of HIR visitors (see + /// `rustc_hir::intravisit`), `D` does not dominate `U`. /// - /// 2. Therefore, `D` is *potentially* storage-dead at `U` (because - /// we might visit `U` without ever getting to `D`). + /// 2. Therefore, `D` is *potentially* storage-dead at `U` (because + /// we might visit `U` without ever getting to `D`). /// - /// 3. However, we guarantee that at each HIR point, each - /// binding/temporary is always either always storage-live - /// or always storage-dead. This is what is being guaranteed - /// by `terminating_scopes` including all blocks where the - /// count of executions is not guaranteed. + /// 3. However, we guarantee that at each HIR point, each + /// binding/temporary is always either always storage-live + /// or always storage-dead. This is what is being guaranteed + /// by `terminating_scopes` including all blocks where the + /// count of executions is not guaranteed. /// - /// 4. By `2.` and `3.`, `D` is *statically* storage-dead at `U`, - /// QED. + /// 4. By `2.` and `3.`, `D` is *statically* storage-dead at `U`, + /// QED. /// /// This property ought to not on (3) in an essential way -- it /// is probably still correct even if we have "unrestricted" terminating diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs index 7e2415fd544..1a206b245d3 100644 --- a/compiler/rustc_middle/src/middle/stability.rs +++ b/compiler/rustc_middle/src/middle/stability.rs @@ -256,24 +256,12 @@ pub enum EvalResult { } // See issue #38412. -fn skip_stability_check_due_to_privacy(tcx: TyCtxt<'_>, mut def_id: DefId) -> bool { - // Check if `def_id` is a trait method. - match tcx.def_kind(def_id) { - DefKind::AssocFn | DefKind::AssocTy | DefKind::AssocConst => { - if let ty::TraitContainer(trait_def_id) = tcx.associated_item(def_id).container { - // Trait methods do not declare visibility (even - // for visibility info in cstore). Use containing - // trait instead, so methods of `pub` traits are - // themselves considered `pub`. - def_id = trait_def_id; - } - } - _ => {} +fn skip_stability_check_due_to_privacy(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + if tcx.def_kind(def_id) == DefKind::TyParam { + // Have no visibility, considered public for the purpose of this check. + return false; } - - let visibility = tcx.visibility(def_id); - - match visibility { + match tcx.visibility(def_id) { // Must check stability for `pub` items. ty::Visibility::Public => false, diff --git a/compiler/rustc_middle/src/mir/coverage/mod.rs b/compiler/rustc_middle/src/mir/coverage.rs index 0421eabc2dc..0421eabc2dc 100644 --- a/compiler/rustc_middle/src/mir/coverage/mod.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index b38efedbf60..b6b6c968501 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -32,7 +32,7 @@ use rustc_target::abi; use rustc_target::asm::InlineAsmRegOrRegClass; use std::borrow::Cow; use std::fmt::{self, Debug, Display, Formatter, Write}; -use std::ops::{Index, IndexMut}; +use std::ops::{ControlFlow, Index, IndexMut}; use std::slice; use std::{iter, mem, option}; @@ -161,7 +161,7 @@ pub struct Body<'tcx> { /// A list of source scopes; these are referenced by statements /// and used for debuginfo. Indexed by a `SourceScope`. - pub source_scopes: IndexVec<SourceScope, SourceScopeData>, + pub source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, /// The yield type of the function, if it is a generator. pub yield_ty: Option<Ty<'tcx>>, @@ -210,16 +210,6 @@ pub struct Body<'tcx> { /// We hold in this field all the constants we are not able to evaluate yet. pub required_consts: Vec<Constant<'tcx>>, - /// The user may be writing e.g. `&[(SOME_CELL, 42)][i].1` and this would get promoted, because - /// we'd statically know that no thing with interior mutability will ever be available to the - /// user without some serious unsafe code. Now this means that our promoted is actually - /// `&[(SOME_CELL, 42)]` and the MIR using it will do the `&promoted[i].1` projection because - /// the index may be a runtime value. Such a promoted value is illegal because it has reachable - /// interior mutability. This flag just makes this situation very obvious where the previous - /// implementation without the flag hid this situation silently. - /// FIXME(oli-obk): rewrite the promoted during promotion to eliminate the cell components. - pub ignore_interior_mut_in_const_validation: bool, - /// Does this body use generic parameters. This is used for the `ConstEvaluatable` check. /// /// Note that this does not actually mean that this body is not computable right now. @@ -244,7 +234,7 @@ impl<'tcx> Body<'tcx> { pub fn new( source: MirSource<'tcx>, basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>, - source_scopes: IndexVec<SourceScope, SourceScopeData>, + source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, local_decls: LocalDecls<'tcx>, user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>, arg_count: usize, @@ -276,7 +266,6 @@ impl<'tcx> Body<'tcx> { var_debug_info, span, required_consts: Vec::new(), - ignore_interior_mut_in_const_validation: false, is_polymorphic: false, predecessor_cache: PredecessorCache::new(), }; @@ -306,7 +295,6 @@ impl<'tcx> Body<'tcx> { required_consts: Vec::new(), generator_kind: None, var_debug_info: Vec::new(), - ignore_interior_mut_in_const_validation: false, is_polymorphic: false, predecessor_cache: PredecessorCache::new(), }; @@ -821,9 +809,6 @@ pub struct LocalDecl<'tcx> { /// flag drop flags to avoid triggering this check as they are introduced /// after typeck. /// - /// Unsafety checking will also ignore dereferences of these locals, - /// so they can be used for raw pointers only used in a desugaring. - /// /// This should be sound because the drop flags are fully algebraic, and /// therefore don't affect the OIBIT or outlives properties of the /// generator. @@ -1010,13 +995,13 @@ impl<'tcx> LocalDecl<'tcx> { } /// Returns `Some` if this is a reference to a static item that is used to - /// access that static + /// access that static. pub fn is_ref_to_static(&self) -> bool { matches!(self.local_info, Some(box LocalInfo::StaticRef { .. })) } - /// Returns `Some` if this is a reference to a static item that is used to - /// access that static + /// Returns `Some` if this is a reference to a thread-local static item that is used to + /// access that static. pub fn is_ref_to_thread_local(&self) -> bool { match self.local_info { Some(box LocalInfo::StaticRef { is_thread_local, .. }) => is_thread_local, @@ -1868,11 +1853,21 @@ rustc_index::newtype_index! { } } -#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)] -pub struct SourceScopeData { +#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable)] +pub struct SourceScopeData<'tcx> { pub span: Span, pub parent_scope: Option<SourceScope>, + /// Whether this scope is the root of a scope tree of another body, + /// inlined into this body by the MIR inliner. + /// `ty::Instance` is the callee, and the `Span` is the call site. + pub inlined: Option<(ty::Instance<'tcx>, Span)>, + + /// Nearest (transitive) parent scope (if any) which is inlined. + /// This is an optimization over walking up `parent_scope` + /// until a scope with `inlined: Some(...)` is found. + pub inlined_parent_scope: Option<SourceScope>, + /// Crate-local information for this source scope, that can't (and /// needn't) be tracked across crates. pub local_data: ClearCrossCrate<SourceScopeLocalData>, @@ -2213,7 +2208,7 @@ impl<'tcx> Debug for Rvalue<'tcx> { let name = ty::tls::with(|tcx| { let mut name = String::new(); - let substs = tcx.lift(&substs).expect("could not lift for printing"); + let substs = tcx.lift(substs).expect("could not lift for printing"); FmtPrinter::new(tcx, &mut name, Namespace::ValueNS) .print_def_path(variant_def.def_id, substs)?; Ok(name) @@ -2236,7 +2231,7 @@ impl<'tcx> Debug for Rvalue<'tcx> { if let Some(def_id) = def_id.as_local() { let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); let name = if tcx.sess.opts.debugging_opts.span_free_formats { - let substs = tcx.lift(&substs).unwrap(); + let substs = tcx.lift(substs).unwrap(); format!( "[closure@{}]", tcx.def_path_str_with_substs(def_id.to_def_id(), substs), @@ -2494,7 +2489,7 @@ impl<'tcx> TypeFoldable<'tcx> for UserTypeProjection { UserTypeProjection { base, projs } } - fn super_visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> bool { + fn super_visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> ControlFlow<()> { self.base.visit_with(visitor) // Note: there's nothing in `self.proj` to visit. } @@ -2530,7 +2525,7 @@ fn pretty_print_const( ) -> fmt::Result { use crate::ty::print::PrettyPrinter; ty::tls::with(|tcx| { - let literal = tcx.lift(&c).unwrap(); + let literal = tcx.lift(c).unwrap(); let mut cx = FmtPrinter::new(tcx, fmt, Namespace::ValueNS); cx.print_alloc_ids = true; cx.pretty_print_const(literal, print_types)?; diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index b9e4f6fb12e..f0bfdae261c 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -152,10 +152,14 @@ impl<'tcx> Rvalue<'tcx> { tcx.mk_ty(ty::Array(operand.ty(local_decls, tcx), count)) } Rvalue::ThreadLocalRef(did) => { + let static_ty = tcx.type_of(did); if tcx.is_mutable_static(did) { - tcx.mk_mut_ptr(tcx.type_of(did)) + tcx.mk_mut_ptr(static_ty) + } else if tcx.is_foreign_item(did) { + tcx.mk_imm_ptr(static_ty) } else { - tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.type_of(did)) + // FIXME: These things don't *really* have 'static lifetime. + tcx.mk_imm_ref(tcx.lifetimes.re_static, static_ty) } } Rvalue::Ref(reg, bk, ref place) => { diff --git a/compiler/rustc_middle/src/mir/terminator/mod.rs b/compiler/rustc_middle/src/mir/terminator.rs index e1071454e65..c9493c67987 100644 --- a/compiler/rustc_middle/src/mir/terminator/mod.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -535,7 +535,7 @@ impl<'tcx> TerminatorKind<'tcx> { Goto { .. } => vec!["".into()], SwitchInt { ref targets, switch_ty, .. } => ty::tls::with(|tcx| { let param_env = ty::ParamEnv::empty(); - let switch_ty = tcx.lift(&switch_ty).unwrap(); + let switch_ty = tcx.lift(switch_ty).unwrap(); let size = tcx.layout_of(param_env.and(switch_ty)).unwrap().size; targets .values diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs index 9297aed66a4..391bd8be7e4 100644 --- a/compiler/rustc_middle/src/mir/type_foldable.rs +++ b/compiler/rustc_middle/src/mir/type_foldable.rs @@ -10,7 +10,6 @@ CloneTypeFoldableAndLiftImpls! { FakeReadCause, RetagKind, SourceScope, - SourceScopeData, SourceScopeLocalData, UserTypeAnnotationIndex, } @@ -88,41 +87,46 @@ impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> { Terminator { source_info: self.source_info, kind } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { use crate::mir::TerminatorKind::*; match self.kind { SwitchInt { ref discr, switch_ty, .. } => { - discr.visit_with(visitor) || switch_ty.visit_with(visitor) + discr.visit_with(visitor)?; + switch_ty.visit_with(visitor) } Drop { ref place, .. } => place.visit_with(visitor), DropAndReplace { ref place, ref value, .. } => { - place.visit_with(visitor) || value.visit_with(visitor) + place.visit_with(visitor)?; + value.visit_with(visitor) } Yield { ref value, .. } => value.visit_with(visitor), Call { ref func, ref args, ref destination, .. } => { - let dest = if let Some((ref loc, _)) = *destination { - loc.visit_with(visitor) - } else { - false + if let Some((ref loc, _)) = *destination { + loc.visit_with(visitor)?; }; - dest || func.visit_with(visitor) || args.visit_with(visitor) + func.visit_with(visitor)?; + args.visit_with(visitor) } Assert { ref cond, ref msg, .. } => { - if cond.visit_with(visitor) { + if cond.visit_with(visitor).is_break() { use AssertKind::*; match msg { BoundsCheck { ref len, ref index } => { - len.visit_with(visitor) || index.visit_with(visitor) + len.visit_with(visitor)?; + index.visit_with(visitor) + } + Overflow(_, l, r) => { + l.visit_with(visitor)?; + r.visit_with(visitor) } - Overflow(_, l, r) => l.visit_with(visitor) || r.visit_with(visitor), OverflowNeg(op) | DivisionByZero(op) | RemainderByZero(op) => { op.visit_with(visitor) } - ResumedAfterReturn(_) | ResumedAfterPanic(_) => false, + ResumedAfterReturn(_) | ResumedAfterPanic(_) => ControlFlow::CONTINUE, } } else { - false + ControlFlow::CONTINUE } } InlineAsm { ref operands, .. } => operands.visit_with(visitor), @@ -133,7 +137,7 @@ impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> { | GeneratorDrop | Unreachable | FalseEdge { .. } - | FalseUnwind { .. } => false, + | FalseUnwind { .. } => ControlFlow::CONTINUE, } } } @@ -143,8 +147,8 @@ impl<'tcx> TypeFoldable<'tcx> for GeneratorKind { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -153,8 +157,9 @@ impl<'tcx> TypeFoldable<'tcx> for Place<'tcx> { Place { local: self.local.fold_with(folder), projection: self.projection.fold_with(folder) } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.local.visit_with(visitor) || self.projection.visit_with(visitor) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.local.visit_with(visitor)?; + self.projection.visit_with(visitor) } } @@ -164,8 +169,8 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<PlaceElem<'tcx>> { folder.tcx().intern_place_elems(&v) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -214,32 +219,47 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { use crate::mir::Rvalue::*; match *self { Use(ref op) => op.visit_with(visitor), Repeat(ref op, _) => op.visit_with(visitor), ThreadLocalRef(did) => did.visit_with(visitor), - Ref(region, _, ref place) => region.visit_with(visitor) || place.visit_with(visitor), + Ref(region, _, ref place) => { + region.visit_with(visitor)?; + place.visit_with(visitor) + } AddressOf(_, ref place) => place.visit_with(visitor), Len(ref place) => place.visit_with(visitor), - Cast(_, ref op, ty) => op.visit_with(visitor) || ty.visit_with(visitor), + Cast(_, ref op, ty) => { + op.visit_with(visitor)?; + ty.visit_with(visitor) + } BinaryOp(_, ref rhs, ref lhs) | CheckedBinaryOp(_, ref rhs, ref lhs) => { - rhs.visit_with(visitor) || lhs.visit_with(visitor) + rhs.visit_with(visitor)?; + lhs.visit_with(visitor) } UnaryOp(_, ref val) => val.visit_with(visitor), Discriminant(ref place) => place.visit_with(visitor), NullaryOp(_, ty) => ty.visit_with(visitor), Aggregate(ref kind, ref fields) => { - (match **kind { - AggregateKind::Array(ty) => ty.visit_with(visitor), - AggregateKind::Tuple => false, + match **kind { + AggregateKind::Array(ty) => { + ty.visit_with(visitor)?; + } + AggregateKind::Tuple => {} AggregateKind::Adt(_, _, substs, user_ty, _) => { - substs.visit_with(visitor) || user_ty.visit_with(visitor) + substs.visit_with(visitor)?; + user_ty.visit_with(visitor)?; + } + AggregateKind::Closure(_, substs) => { + substs.visit_with(visitor)?; } - AggregateKind::Closure(_, substs) => substs.visit_with(visitor), - AggregateKind::Generator(_, substs, _) => substs.visit_with(visitor), - }) || fields.visit_with(visitor) + AggregateKind::Generator(_, substs, _) => { + substs.visit_with(visitor)?; + } + } + fields.visit_with(visitor) } } } @@ -254,7 +274,7 @@ impl<'tcx> TypeFoldable<'tcx> for Operand<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { match *self { Operand::Copy(ref place) | Operand::Move(ref place) => place.visit_with(visitor), Operand::Constant(ref c) => c.visit_with(visitor), @@ -278,13 +298,13 @@ impl<'tcx> TypeFoldable<'tcx> for PlaceElem<'tcx> { } } - fn super_visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> bool { + fn super_visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> ControlFlow<()> { use crate::mir::ProjectionElem::*; match self { Field(_, ty) => ty.visit_with(visitor), Index(v) => v.visit_with(visitor), - _ => false, + _ => ControlFlow::CONTINUE, } } } @@ -293,8 +313,8 @@ impl<'tcx> TypeFoldable<'tcx> for Field { fn super_fold_with<F: TypeFolder<'tcx>>(&self, _: &mut F) -> Self { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -302,8 +322,8 @@ impl<'tcx> TypeFoldable<'tcx> for GeneratorSavedLocal { fn super_fold_with<F: TypeFolder<'tcx>>(&self, _: &mut F) -> Self { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -311,8 +331,8 @@ impl<'tcx, R: Idx, C: Idx> TypeFoldable<'tcx> for BitMatrix<R, C> { fn super_fold_with<F: TypeFolder<'tcx>>(&self, _: &mut F) -> Self { self.clone() } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -324,7 +344,7 @@ impl<'tcx> TypeFoldable<'tcx> for Constant<'tcx> { literal: self.literal.fold_with(folder), } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { self.literal.visit_with(visitor) } } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 58dd0bc00d2..d8d639ab734 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -85,7 +85,7 @@ macro_rules! make_mir_visitor { } fn visit_source_scope_data(&mut self, - scope_data: & $($mutability)? SourceScopeData) { + scope_data: & $($mutability)? SourceScopeData<'tcx>) { self.super_source_scope_data(scope_data); } @@ -317,10 +317,15 @@ macro_rules! make_mir_visitor { } } - fn super_source_scope_data(&mut self, scope_data: & $($mutability)? SourceScopeData) { + fn super_source_scope_data( + &mut self, + scope_data: & $($mutability)? SourceScopeData<'tcx>, + ) { let SourceScopeData { span, parent_scope, + inlined, + inlined_parent_scope, local_data: _, } = scope_data; @@ -328,6 +333,34 @@ macro_rules! make_mir_visitor { if let Some(parent_scope) = parent_scope { self.visit_source_scope(parent_scope); } + if let Some((callee, callsite_span)) = inlined { + let location = START_BLOCK.start_location(); + + self.visit_span(callsite_span); + + let ty::Instance { def: callee_def, substs: callee_substs } = callee; + match callee_def { + ty::InstanceDef::Item(_def_id) => {} + + ty::InstanceDef::Intrinsic(_def_id) | + ty::InstanceDef::VtableShim(_def_id) | + ty::InstanceDef::ReifyShim(_def_id) | + ty::InstanceDef::Virtual(_def_id, _) | + ty::InstanceDef::ClosureOnceShim { call_once: _def_id } | + ty::InstanceDef::DropGlue(_def_id, None) => {} + + ty::InstanceDef::FnPtrShim(_def_id, ty) | + ty::InstanceDef::DropGlue(_def_id, Some(ty)) | + ty::InstanceDef::CloneShim(_def_id, ty) => { + // FIXME(eddyb) use a better `TyContext` here. + self.visit_ty(ty, TyContext::Location(location)); + } + } + self.visit_substs(callee_substs, location); + } + if let Some(inlined_parent_scope) = inlined_parent_scope { + self.visit_source_scope(inlined_parent_scope); + } } fn super_statement(&mut self, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 1acb44f6d22..89faa97d50f 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1267,6 +1267,7 @@ rustc_queries! { TypeChecking { query visibility(def_id: DefId) -> ty::Visibility { + eval_always desc { |tcx| "computing visibility of `{}`", tcx.def_path_str(def_id) } } } diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs index 1dd6d590d90..4deb7225dcb 100644 --- a/compiler/rustc_middle/src/traits/mod.rs +++ b/compiler/rustc_middle/src/traits/mod.rs @@ -13,6 +13,7 @@ use crate::mir::interpret::ErrorHandled; use crate::ty::subst::SubstsRef; use crate::ty::{self, AdtKind, Ty, TyCtxt}; +use rustc_errors::{Applicability, DiagnosticBuilder}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_span::symbol::Symbol; @@ -339,10 +340,24 @@ impl ObligationCauseCode<'_> { #[cfg(target_arch = "x86_64")] static_assert_size!(ObligationCauseCode<'_>, 32); +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum StatementAsExpression { + CorrectType, + NeedsBoxing, +} + +impl<'tcx> ty::Lift<'tcx> for StatementAsExpression { + type Lifted = StatementAsExpression; + fn lift_to_tcx(self, _tcx: TyCtxt<'tcx>) -> Option<StatementAsExpression> { + Some(self) + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)] pub struct MatchExpressionArmCause<'tcx> { pub arm_span: Span, - pub semi_span: Option<Span>, + pub scrut_span: Span, + pub semi_span: Option<(Span, StatementAsExpression)>, pub source: hir::MatchSource, pub prior_arms: Vec<Span>, pub last_ty: Ty<'tcx>, @@ -355,7 +370,7 @@ pub struct IfExpressionCause { pub then: Span, pub else_sp: Span, pub outer: Option<Span>, - pub semicolon: Option<Span>, + pub semicolon: Option<(Span, StatementAsExpression)>, pub opt_suggest_box_span: Option<Span>, } @@ -646,13 +661,13 @@ impl ObjectSafetyViolation { ObjectSafetyViolation::SizedSelf(_) => "it requires `Self: Sized`".into(), ObjectSafetyViolation::SupertraitSelf(ref spans) => { if spans.iter().any(|sp| *sp != DUMMY_SP) { - "it uses `Self` as a type parameter in this".into() + "it uses `Self` as a type parameter".into() } else { "it cannot use `Self` as a type parameter in a supertrait or `where`-clause" .into() } } - ObjectSafetyViolation::Method(name, MethodViolationCode::StaticMethod(_), _) => { + ObjectSafetyViolation::Method(name, MethodViolationCode::StaticMethod(_, _, _), _) => { format!("associated function `{}` has no `self` parameter", name).into() } ObjectSafetyViolation::Method( @@ -686,32 +701,65 @@ impl ObjectSafetyViolation { } } - pub fn solution(&self) -> Option<(String, Option<(String, Span)>)> { - Some(match *self { - ObjectSafetyViolation::SizedSelf(_) | ObjectSafetyViolation::SupertraitSelf(_) => { - return None; + pub fn solution(&self, err: &mut DiagnosticBuilder<'_>) { + match *self { + ObjectSafetyViolation::SizedSelf(_) | ObjectSafetyViolation::SupertraitSelf(_) => {} + ObjectSafetyViolation::Method( + name, + MethodViolationCode::StaticMethod(sugg, self_span, has_args), + _, + ) => { + err.span_suggestion( + self_span, + &format!( + "consider turning `{}` into a method by giving it a `&self` argument", + name + ), + format!("&self{}", if has_args { ", " } else { "" }), + Applicability::MaybeIncorrect, + ); + match sugg { + Some((sugg, span)) => { + err.span_suggestion( + span, + &format!( + "alternatively, consider constraining `{}` so it does not apply to \ + trait objects", + name + ), + sugg.to_string(), + Applicability::MaybeIncorrect, + ); + } + None => { + err.help(&format!( + "consider turning `{}` into a method by giving it a `&self` \ + argument or constraining it so it does not apply to trait objects", + name + )); + } + } } - ObjectSafetyViolation::Method(name, MethodViolationCode::StaticMethod(sugg), _) => ( - format!( - "consider turning `{}` into a method by giving it a `&self` argument or \ - constraining it so it does not apply to trait objects", - name - ), - sugg.map(|(sugg, sp)| (sugg.to_string(), sp)), - ), ObjectSafetyViolation::Method( name, MethodViolationCode::UndispatchableReceiver, span, - ) => ( - format!("consider changing method `{}`'s `self` parameter to be `&self`", name), - Some(("&Self".to_string(), span)), - ), + ) => { + err.span_suggestion( + span, + &format!( + "consider changing method `{}`'s `self` parameter to be `&self`", + name + ), + "&Self".to_string(), + Applicability::MachineApplicable, + ); + } ObjectSafetyViolation::AssocConst(name, _) | ObjectSafetyViolation::Method(name, ..) => { - (format!("consider moving `{}` to another trait", name), None) + err.help(&format!("consider moving `{}` to another trait", name)); } - }) + } } pub fn spans(&self) -> SmallVec<[Span; 1]> { @@ -735,7 +783,7 @@ impl ObjectSafetyViolation { #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)] pub enum MethodViolationCode { /// e.g., `fn foo()` - StaticMethod(Option<(&'static str, Span)>), + StaticMethod(Option<(&'static str, Span)>, Span, bool /* has args */), /// e.g., `fn foo(&self, x: Self)` ReferencesSelfInput(usize), diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs index 358ead507b4..c570ad3273d 100644 --- a/compiler/rustc_middle/src/traits/select.rs +++ b/compiler/rustc_middle/src/traits/select.rs @@ -127,7 +127,10 @@ pub enum SelectionCandidate<'tcx> { TraitAliasCandidate(DefId), - ObjectCandidate, + /// Matching `dyn Trait` with a supertrait of `Trait`. The index is the + /// position in the iterator returned by + /// `rustc_infer::traits::util::supertraits`. + ObjectCandidate(usize), BuiltinObjectCandidate, diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index e24ba6d7a1e..f6ea6743a0e 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -22,7 +22,7 @@ use crate::ty::{ ExistentialPredicate, FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy, IntVar, IntVid, List, ParamConst, ParamTy, PolyFnSig, Predicate, PredicateInner, PredicateKind, ProjectionTy, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, - TyVid, TypeAndMut, + TyVid, TypeAndMut, Visibility, }; use rustc_ast as ast; use rustc_ast::expand::allocator::AllocatorKind; @@ -134,7 +134,7 @@ impl<'tcx> CtxtInterners<'tcx> { fn intern_predicate(&self, kind: PredicateKind<'tcx>) -> &'tcx PredicateInner<'tcx> { self.predicate .intern(kind, |kind| { - let flags = super::flags::FlagComputation::for_predicate(&kind); + let flags = super::flags::FlagComputation::for_predicate(kind); let predicate_struct = PredicateInner { kind, @@ -911,6 +911,9 @@ pub struct GlobalCtxt<'tcx> { /// Common consts, pre-interned for your convenience. pub consts: CommonConsts<'tcx>, + /// Visibilities produced by resolver. + pub visibilities: FxHashMap<LocalDefId, Visibility>, + /// Resolutions of `extern crate` items produced by resolver. extern_crate_map: FxHashMap<LocalDefId, CrateNum>, @@ -1057,7 +1060,7 @@ impl<'tcx> TyCtxt<'tcx> { ) } - pub fn lift<T: ?Sized + Lift<'tcx>>(self, value: &T) -> Option<T::Lifted> { + pub fn lift<T: Lift<'tcx>>(self, value: T) -> Option<T::Lifted> { value.lift_to_tcx(self) } @@ -1124,6 +1127,7 @@ impl<'tcx> TyCtxt<'tcx> { types: common_types, lifetimes: common_lifetimes, consts: common_consts, + visibilities: resolutions.visibilities, extern_crate_map: resolutions.extern_crate_map, trait_map, export_map: resolutions.export_map, @@ -1565,16 +1569,16 @@ impl<'tcx> TyCtxt<'tcx> { /// e.g., `()` or `u8`, was interned in a different context. pub trait Lift<'tcx>: fmt::Debug { type Lifted: fmt::Debug + 'tcx; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted>; + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted>; } macro_rules! nop_lift { ($set:ident; $ty:ty => $lifted:ty) => { impl<'a, 'tcx> Lift<'tcx> for $ty { type Lifted = $lifted; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - if tcx.interners.$set.contains_pointer_to(&Interned(*self)) { - Some(unsafe { mem::transmute(*self) }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + if tcx.interners.$set.contains_pointer_to(&Interned(self)) { + Some(unsafe { mem::transmute(self) }) } else { None } @@ -1587,12 +1591,12 @@ macro_rules! nop_list_lift { ($set:ident; $ty:ty => $lifted:ty) => { impl<'a, 'tcx> Lift<'tcx> for &'a List<$ty> { type Lifted = &'tcx List<$lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { if self.is_empty() { return Some(List::empty()); } - if tcx.interners.$set.contains_pointer_to(&Interned(*self)) { - Some(unsafe { mem::transmute(*self) }) + if tcx.interners.$set.contains_pointer_to(&Interned(self)) { + Some(unsafe { mem::transmute(self) }) } else { None } @@ -2032,13 +2036,13 @@ direct_interners! { macro_rules! slice_interners { ($($field:ident: $method:ident($ty:ty)),+ $(,)?) => ( - $(impl<'tcx> TyCtxt<'tcx> { - pub fn $method(self, v: &[$ty]) -> &'tcx List<$ty> { + impl<'tcx> TyCtxt<'tcx> { + $(pub fn $method(self, v: &[$ty]) -> &'tcx List<$ty> { self.interners.$field.intern_ref(v, || { Interned(List::from_arena(&*self.arena, v)) }).0 - } - })+ + })+ + } ); } diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs index 82d698b37ab..5ec0ec0c56a 100644 --- a/compiler/rustc_middle/src/ty/error.rs +++ b/compiler/rustc_middle/src/ty/error.rs @@ -229,7 +229,7 @@ impl<'tcx> ty::TyS<'tcx> { ty::Adt(def, _) => format!("{} `{}`", def.descr(), tcx.def_path_str(def.did)).into(), ty::Foreign(def_id) => format!("extern type `{}`", tcx.def_path_str(def_id)).into(), ty::Array(t, n) => { - let n = tcx.lift(&n).unwrap(); + let n = tcx.lift(n).unwrap(); match n.try_eval_usize(tcx, ty::ParamEnv::empty()) { _ if t.is_simple_ty() => format!("array `{}`", self).into(), Some(n) => format!("array of {} element{}", n, pluralize!(n)).into(), @@ -334,26 +334,15 @@ impl<'tcx> TyCtxt<'tcx> { debug!("note_and_explain_type_err err={:?} cause={:?}", err, cause); match err { Sorts(values) => { - let expected_str = values.expected.sort_string(self); - let found_str = values.found.sort_string(self); - if expected_str == found_str && expected_str == "closure" { - db.note("no two closures, even if identical, have the same type"); - db.help("consider boxing your closure and/or using it as a trait object"); - } - if expected_str == found_str && expected_str == "opaque type" { - // Issue #63167 - db.note("distinct uses of `impl Trait` result in different opaque types"); - let e_str = values.expected.to_string(); - let f_str = values.found.to_string(); - if e_str == f_str && &e_str == "impl std::future::Future" { - // FIXME: use non-string based check. - db.help( - "if both `Future`s have the same `Output` type, consider \ - `.await`ing on both of them", - ); - } - } match (values.expected.kind(), values.found.kind()) { + (ty::Closure(..), ty::Closure(..)) => { + db.note("no two closures, even if identical, have the same type"); + db.help("consider boxing your closure and/or using it as a trait object"); + } + (ty::Opaque(..), ty::Opaque(..)) => { + // Issue #63167 + db.note("distinct uses of `impl Trait` result in different opaque types"); + } (ty::Float(_), ty::Infer(ty::IntVar(_))) => { if let Ok( // Issue #53280 @@ -382,12 +371,12 @@ impl<'tcx> TyCtxt<'tcx> { } db.note( "a type parameter was expected, but a different one was found; \ - you might be missing a type parameter or trait bound", + you might be missing a type parameter or trait bound", ); db.note( "for more information, visit \ - https://doc.rust-lang.org/book/ch10-02-traits.html\ - #traits-as-parameters", + https://doc.rust-lang.org/book/ch10-02-traits.html\ + #traits-as-parameters", ); } (ty::Projection(_), ty::Projection(_)) => { @@ -471,8 +460,8 @@ impl<T> Trait<T> for X { } db.note( "for more information, visit \ - https://doc.rust-lang.org/book/ch10-02-traits.html\ - #traits-as-parameters", + https://doc.rust-lang.org/book/ch10-02-traits.html\ + #traits-as-parameters", ); } (ty::Param(p), ty::Closure(..) | ty::Generator(..)) => { diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs index c9a4022330a..8b97a87f214 100644 --- a/compiler/rustc_middle/src/ty/flags.rs +++ b/compiler/rustc_middle/src/ty/flags.rs @@ -22,7 +22,7 @@ impl FlagComputation { result } - pub fn for_predicate(kind: &ty::PredicateKind<'_>) -> FlagComputation { + pub fn for_predicate(kind: ty::PredicateKind<'_>) -> FlagComputation { let mut result = FlagComputation::new(); result.add_predicate_kind(kind); result @@ -53,7 +53,14 @@ impl FlagComputation { /// Adds the flags/depth from a set of types that appear within the current type, but within a /// region binder. - fn add_bound_computation(&mut self, computation: FlagComputation) { + fn bound_computation<T, F>(&mut self, value: ty::Binder<T>, f: F) + where + F: FnOnce(&mut Self, T), + { + let mut computation = FlagComputation::new(); + + f(&mut computation, value.skip_binder()); + self.add_flags(computation.flags); // The types that contributed to `computation` occurred within @@ -101,9 +108,7 @@ impl FlagComputation { } &ty::GeneratorWitness(ts) => { - let mut computation = FlagComputation::new(); - computation.add_tys(ts.skip_binder()); - self.add_bound_computation(computation); + self.bound_computation(ts, |flags, ts| flags.add_tys(ts)); } &ty::Closure(_, substs) => { @@ -154,20 +159,21 @@ impl FlagComputation { self.add_substs(substs); } - &ty::Dynamic(ref obj, r) => { - let mut computation = FlagComputation::new(); - for predicate in obj.skip_binder().iter() { - match predicate { - ty::ExistentialPredicate::Trait(tr) => computation.add_substs(tr.substs), - ty::ExistentialPredicate::Projection(p) => { - let mut proj_computation = FlagComputation::new(); - proj_computation.add_existential_projection(&p); - self.add_bound_computation(proj_computation); + &ty::Dynamic(obj, r) => { + self.bound_computation(obj, |computation, obj| { + for predicate in obj.iter() { + match predicate { + ty::ExistentialPredicate::Trait(tr) => { + computation.add_substs(tr.substs) + } + ty::ExistentialPredicate::Projection(p) => { + computation.add_existential_projection(&p); + } + ty::ExistentialPredicate::AutoTrait(_) => {} } - ty::ExistentialPredicate::AutoTrait(_) => {} } - } - self.add_bound_computation(computation); + }); + self.add_region(r); } @@ -195,22 +201,21 @@ impl FlagComputation { self.add_substs(substs); } - &ty::FnPtr(f) => { - self.add_fn_sig(f); - } + &ty::FnPtr(fn_sig) => self.bound_computation(fn_sig, |computation, fn_sig| { + computation.add_tys(fn_sig.inputs()); + computation.add_ty(fn_sig.output()); + }), } } - fn add_predicate_kind(&mut self, kind: &ty::PredicateKind<'_>) { + fn add_predicate_kind(&mut self, kind: ty::PredicateKind<'_>) { match kind { ty::PredicateKind::ForAll(binder) => { - let mut computation = FlagComputation::new(); - - computation.add_predicate_atom(binder.skip_binder()); - - self.add_bound_computation(computation); + self.bound_computation(binder, |computation, atom| { + computation.add_predicate_atom(atom) + }); } - &ty::PredicateKind::Atom(atom) => self.add_predicate_atom(atom), + ty::PredicateKind::Atom(atom) => self.add_predicate_atom(atom), } } @@ -266,15 +271,6 @@ impl FlagComputation { } } - fn add_fn_sig(&mut self, fn_sig: ty::PolyFnSig<'_>) { - let mut computation = FlagComputation::new(); - - computation.add_tys(fn_sig.skip_binder().inputs()); - computation.add_ty(fn_sig.skip_binder().output()); - - self.add_bound_computation(computation); - } - fn add_region(&mut self, r: ty::Region<'_>) { self.add_flags(r.type_flags()); if let ty::ReLateBound(debruijn, _) = *r { diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs index 5524d91a6d5..70a8157c04a 100644 --- a/compiler/rustc_middle/src/ty/fold.rs +++ b/compiler/rustc_middle/src/ty/fold.rs @@ -30,8 +30,6 @@ //! //! These methods return true to indicate that the visitor has found what it is //! looking for, and does not need to visit anything else. - -use crate::ty::structural_impls::PredicateVisitor; use crate::ty::{self, flags::FlagComputation, Binder, Ty, TyCtxt, TypeFlags}; use rustc_hir as hir; use rustc_hir::def_id::DefId; @@ -39,6 +37,7 @@ use rustc_hir::def_id::DefId; use rustc_data_structures::fx::FxHashSet; use std::collections::BTreeMap; use std::fmt; +use std::ops::ControlFlow; /// This trait is implemented for every type that can be folded. /// Basically, every type that has a corresponding method in `TypeFolder`. @@ -50,8 +49,8 @@ pub trait TypeFoldable<'tcx>: fmt::Debug + Clone { self.super_fold_with(folder) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool; - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()>; + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { self.super_visit_with(visitor) } @@ -60,7 +59,7 @@ pub trait TypeFoldable<'tcx>: fmt::Debug + Clone { /// If `binder` is `ty::INNERMOST`, this indicates whether /// there are any late-bound regions that appear free. fn has_vars_bound_at_or_above(&self, binder: ty::DebruijnIndex) -> bool { - self.visit_with(&mut HasEscapingVarsVisitor { outer_index: binder }) + self.visit_with(&mut HasEscapingVarsVisitor { outer_index: binder }).is_break() } /// Returns `true` if this `self` has any regions that escape `binder` (and @@ -74,7 +73,7 @@ pub trait TypeFoldable<'tcx>: fmt::Debug + Clone { } fn has_type_flags(&self, flags: TypeFlags) -> bool { - self.visit_with(&mut HasTypeFlagsVisitor { flags }) + self.visit_with(&mut HasTypeFlagsVisitor { flags }).is_break() } fn has_projections(&self) -> bool { self.has_type_flags(TypeFlags::HAS_PROJECTION) @@ -145,11 +144,11 @@ pub trait TypeFoldable<'tcx>: fmt::Debug + Clone { } /// A visitor that does not recurse into types, works like `fn walk_shallow` in `Ty`. - fn visit_tys_shallow(&self, visit: impl FnMut(Ty<'tcx>) -> bool) -> bool { + fn visit_tys_shallow(&self, visit: impl FnMut(Ty<'tcx>) -> ControlFlow<()>) -> ControlFlow<()> { pub struct Visitor<F>(F); - impl<'tcx, F: FnMut(Ty<'tcx>) -> bool> TypeVisitor<'tcx> for Visitor<F> { - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + impl<'tcx, F: FnMut(Ty<'tcx>) -> ControlFlow<()>> TypeVisitor<'tcx> for Visitor<F> { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { self.0(ty) } } @@ -162,8 +161,8 @@ impl TypeFoldable<'tcx> for hir::Constness { fn super_fold_with<F: TypeFolder<'tcx>>(&self, _: &mut F) -> Self { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -196,21 +195,25 @@ pub trait TypeFolder<'tcx>: Sized { } pub trait TypeVisitor<'tcx>: Sized { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<()> { t.super_visit_with(self) } - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { t.super_visit_with(self) } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { r.super_visit_with(self) } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { c.super_visit_with(self) } + + fn visit_predicate(&mut self, p: ty::Predicate<'tcx>) -> ControlFlow<()> { + p.super_visit_with(self) + } } /////////////////////////////////////////////////////////////////////////// @@ -300,8 +303,6 @@ impl<'tcx> TyCtxt<'tcx> { value: &impl TypeFoldable<'tcx>, callback: impl FnMut(ty::Region<'tcx>) -> bool, ) -> bool { - return value.visit_with(&mut RegionVisitor { outer_index: ty::INNERMOST, callback }); - struct RegionVisitor<F> { /// The index of a binder *just outside* the things we have /// traversed. If we encounter a bound region bound by this @@ -328,31 +329,39 @@ impl<'tcx> TyCtxt<'tcx> { where F: FnMut(ty::Region<'tcx>) -> bool, { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<()> { self.outer_index.shift_in(1); let result = t.as_ref().skip_binder().visit_with(self); self.outer_index.shift_out(1); result } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { match *r { ty::ReLateBound(debruijn, _) if debruijn < self.outer_index => { - false // ignore bound regions, keep visiting + ControlFlow::CONTINUE + } + _ => { + if (self.callback)(r) { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } - _ => (self.callback)(r), } } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { // We're only interested in types involving regions if ty.flags().intersects(TypeFlags::HAS_FREE_REGIONS) { ty.super_visit_with(self) } else { - false // keep visiting + ControlFlow::CONTINUE } } } + + value.visit_with(&mut RegionVisitor { outer_index: ty::INNERMOST, callback }).is_break() } } @@ -668,7 +677,7 @@ impl<'tcx> TyCtxt<'tcx> { { let mut collector = LateBoundRegionsCollector::new(just_constraint); let result = value.as_ref().skip_binder().visit_with(&mut collector); - assert!(!result); // should never have stopped early + assert!(result.is_continue()); // should never have stopped early collector.regions } @@ -682,7 +691,7 @@ impl<'tcx> TyCtxt<'tcx> { } /// Rewrite any late-bound regions so that they are anonymous. Region numbers are - /// assigned starting at 1 and increasing monotonically in the order traversed + /// assigned starting at 0 and increasing monotonically in the order traversed /// by the fold operation. /// /// The chief purpose of this function is to canonicalize regions so that two @@ -696,8 +705,9 @@ impl<'tcx> TyCtxt<'tcx> { let mut counter = 0; Binder::bind( self.replace_late_bound_regions(sig, |_| { + let r = self.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BrAnon(counter))); counter += 1; - self.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BrAnon(counter))) + r }) .0, ) @@ -834,45 +844,55 @@ struct HasEscapingVarsVisitor { } impl<'tcx> TypeVisitor<'tcx> for HasEscapingVarsVisitor { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<()> { self.outer_index.shift_in(1); let result = t.super_visit_with(self); self.outer_index.shift_out(1); result } - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { // If the outer-exclusive-binder is *strictly greater* than // `outer_index`, that means that `t` contains some content // bound at `outer_index` or above (because // `outer_exclusive_binder` is always 1 higher than the // content in `t`). Therefore, `t` has some escaping vars. - t.outer_exclusive_binder > self.outer_index + if t.outer_exclusive_binder > self.outer_index { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { // If the region is bound by `outer_index` or anything outside // of outer index, then it escapes the binders we have // visited. - r.bound_at_or_above_binder(self.outer_index) + if r.bound_at_or_above_binder(self.outer_index) { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } - fn visit_const(&mut self, ct: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, ct: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { // we don't have a `visit_infer_const` callback, so we have to // hook in here to catch this case (annoying...), but // otherwise we do want to remember to visit the rest of the // const, as it has types/regions embedded in a lot of other // places. match ct.val { - ty::ConstKind::Bound(debruijn, _) if debruijn >= self.outer_index => true, + ty::ConstKind::Bound(debruijn, _) if debruijn >= self.outer_index => ControlFlow::BREAK, _ => ct.super_visit_with(self), } } -} -impl<'tcx> PredicateVisitor<'tcx> for HasEscapingVarsVisitor { - fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool { - predicate.inner.outer_exclusive_binder > self.outer_index + fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ControlFlow<()> { + if predicate.inner.outer_exclusive_binder > self.outer_index { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } } @@ -882,38 +902,41 @@ struct HasTypeFlagsVisitor { } impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor { - fn visit_ty(&mut self, t: Ty<'_>) -> bool { + fn visit_ty(&mut self, t: Ty<'_>) -> ControlFlow<()> { debug!( "HasTypeFlagsVisitor: t={:?} t.flags={:?} self.flags={:?}", t, t.flags(), self.flags ); - t.flags().intersects(self.flags) + if t.flags().intersects(self.flags) { ControlFlow::BREAK } else { ControlFlow::CONTINUE } } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { let flags = r.type_flags(); debug!("HasTypeFlagsVisitor: r={:?} r.flags={:?} self.flags={:?}", r, flags, self.flags); - flags.intersects(self.flags) + if flags.intersects(self.flags) { ControlFlow::BREAK } else { ControlFlow::CONTINUE } } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { let flags = FlagComputation::for_const(c); debug!("HasTypeFlagsVisitor: c={:?} c.flags={:?} self.flags={:?}", c, flags, self.flags); - flags.intersects(self.flags) + if flags.intersects(self.flags) { ControlFlow::BREAK } else { ControlFlow::CONTINUE } } -} -impl<'tcx> PredicateVisitor<'tcx> for HasTypeFlagsVisitor { - fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool { + fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ControlFlow<()> { debug!( "HasTypeFlagsVisitor: predicate={:?} predicate.flags={:?} self.flags={:?}", predicate, predicate.inner.flags, self.flags ); - predicate.inner.flags.intersects(self.flags) + if predicate.inner.flags.intersects(self.flags) { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } } + /// Collects all the late-bound regions at the innermost binding level /// into a hash set. struct LateBoundRegionsCollector { @@ -941,45 +964,45 @@ impl LateBoundRegionsCollector { } impl<'tcx> TypeVisitor<'tcx> for LateBoundRegionsCollector { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<()> { self.current_index.shift_in(1); let result = t.super_visit_with(self); self.current_index.shift_out(1); result } - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { // if we are only looking for "constrained" region, we have to // ignore the inputs to a projection, as they may not appear // in the normalized form if self.just_constrained { if let ty::Projection(..) | ty::Opaque(..) = t.kind() { - return false; + return ControlFlow::CONTINUE; } } t.super_visit_with(self) } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { // if we are only looking for "constrained" region, we have to // ignore the inputs of an unevaluated const, as they may not appear // in the normalized form if self.just_constrained { if let ty::ConstKind::Unevaluated(..) = c.val { - return false; + return ControlFlow::CONTINUE; } } c.super_visit_with(self) } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { if let ty::ReLateBound(debruijn, br) = *r { if debruijn == self.current_index { self.regions.insert(br); } } - false + ControlFlow::CONTINUE } } diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs index bf1f5b81c9f..2f7707b9498 100644 --- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs +++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs @@ -201,13 +201,13 @@ impl<'tcx> TyS<'tcx> { ), Array(ty, len) => match len.try_eval_usize(tcx, param_env) { + Some(0) | None => DefIdForest::empty(), // If the array is definitely non-empty, it's uninhabited if // the type of its elements is uninhabited. - Some(n) if n != 0 => ty.uninhabited_from(tcx, param_env), - _ => DefIdForest::empty(), + Some(1..) => ty.uninhabited_from(tcx, param_env), }, - // References to uninitialised memory is valid for any type, including + // References to uninitialised memory are valid for any type, including // uninhabited types, in unsafe code, so we treat all references as // inhabited. // The precise semantics of inhabitedness with respect to references is currently diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index 4a2c97b4a40..8b3fb875070 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -258,7 +258,7 @@ impl<'tcx> InstanceDef<'tcx> { impl<'tcx> fmt::Display for Instance<'tcx> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { ty::tls::with(|tcx| { - let substs = tcx.lift(&self.substs).expect("could not lift for printing"); + let substs = tcx.lift(self.substs).expect("could not lift for printing"); FmtPrinter::new(tcx, &mut *f, Namespace::ValueNS) .print_def_path(self.def_id(), substs)?; Ok(()) diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index cc378f1ea47..096c53ca996 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -46,7 +46,7 @@ use std::cell::RefCell; use std::cmp::Ordering; use std::fmt; use std::hash::{Hash, Hasher}; -use std::ops::Range; +use std::ops::{ControlFlow, Range}; use std::ptr; use std::str; @@ -125,6 +125,7 @@ mod sty; pub struct ResolverOutputs { pub definitions: rustc_hir::definitions::Definitions, pub cstore: Box<CrateStoreDyn>, + pub visibilities: FxHashMap<LocalDefId, Visibility>, pub extern_crate_map: FxHashMap<LocalDefId, CrateNum>, pub maybe_unused_trait_imports: FxHashSet<LocalDefId>, pub maybe_unused_extern_crates: Vec<(LocalDefId, Span)>, @@ -265,6 +266,10 @@ impl<'tcx> AssociatedItems<'tcx> { self.items.iter().map(|(_, v)| *v) } + pub fn len(&self) -> usize { + self.items.len() + } + /// Returns an iterator over all associated items with the given name, ignoring hygiene. pub fn filter_by_name_unhygienic( &self, @@ -1771,8 +1776,9 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> { ParamEnv::new(self.caller_bounds().fold_with(folder), self.reveal().fold_with(folder)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.caller_bounds().visit_with(visitor) || self.reveal().visit_with(visitor) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.caller_bounds().visit_with(visitor)?; + self.reveal().visit_with(visitor) } } diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index acdf1f642ab..fa38bddcae2 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -22,7 +22,7 @@ use std::cell::Cell; use std::char; use std::collections::BTreeMap; use std::fmt::{self, Write as _}; -use std::ops::{Deref, DerefMut}; +use std::ops::{ControlFlow, Deref, DerefMut}; // `pretty` is a separate module only for organization. use super::*; @@ -658,7 +658,7 @@ pub trait PrettyPrinter<'tcx>: let span = self.tcx().hir().span(hir_id); p!(write("@{}", self.tcx().sess.source_map().span_to_string(span))); } else { - p!(write("@{}", self.tcx().def_path_str(did))); + p!(write("@"), print_def_path(did, substs)); } } else { p!(print_def_path(did, substs)); @@ -694,7 +694,7 @@ pub trait PrettyPrinter<'tcx>: p!(write("@{}", self.tcx().sess.source_map().span_to_string(span))); } } else { - p!(write("@{}", self.tcx().def_path_str(did))); + p!(write("@"), print_def_path(did, substs)); } } else { p!(print_def_path(did, substs)); @@ -1803,7 +1803,7 @@ impl<F: fmt::Write> FmtPrinter<'_, 'tcx, F> { { struct LateBoundRegionNameCollector<'a>(&'a mut FxHashSet<Symbol>); impl<'tcx> ty::fold::TypeVisitor<'tcx> for LateBoundRegionNameCollector<'_> { - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { if let ty::ReLateBound(_, ty::BrNamed(_, name)) = *r { self.0.insert(name); } @@ -1848,7 +1848,7 @@ macro_rules! forward_display_to_print { $(impl fmt::Display for $ty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { ty::tls::with(|tcx| { - tcx.lift(self) + tcx.lift(*self) .expect("could not lift for printing") .print(FmtPrinter::new(tcx, f, Namespace::TypeNS))?; Ok(()) diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs index d3a7412ef14..7ba4d5a14df 100644 --- a/compiler/rustc_middle/src/ty/query/mod.rs +++ b/compiler/rustc_middle/src/ty/query/mod.rs @@ -34,7 +34,6 @@ use crate::ty::util::AlwaysRequiresDrop; use crate::ty::{self, AdtSizedConstraint, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap}; -use rustc_data_structures::profiling::ProfileCategory::*; use rustc_data_structures::stable_hasher::StableVec; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; @@ -169,26 +168,71 @@ pub fn force_from_dep_node<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> bool return false; } - rustc_dep_node_force!([dep_node, tcx] - // These are inputs that are expected to be pre-allocated and that - // should therefore always be red or green already. - DepKind::CrateMetadata | + macro_rules! force_from_dep_node { + ($($(#[$attr:meta])* [$($modifiers:tt)*] $name:ident($K:ty),)*) => { + match dep_node.kind { + // These are inputs that are expected to be pre-allocated and that + // should therefore always be red or green already. + DepKind::CrateMetadata | - // These are anonymous nodes. - DepKind::TraitSelect | + // These are anonymous nodes. + DepKind::TraitSelect | - // We don't have enough information to reconstruct the query key of - // these. - DepKind::CompileCodegenUnit => { - bug!("force_from_dep_node: encountered {:?}", dep_node) + // We don't have enough information to reconstruct the query key of + // these. + DepKind::CompileCodegenUnit | + + // Forcing this makes no sense. + DepKind::Null => { + bug!("force_from_dep_node: encountered {:?}", dep_node) + } + + $(DepKind::$name => { + debug_assert!(<$K as DepNodeParams<TyCtxt<'_>>>::can_reconstruct_query_key()); + + if let Some(key) = <$K as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node) { + force_query::<queries::$name<'_>, _>( + tcx, + key, + DUMMY_SP, + *dep_node + ); + return true; + } + })* + } } - ); + } + + rustc_dep_node_append! { [force_from_dep_node!][] } false } pub(crate) fn try_load_from_on_disk_cache<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) { - rustc_dep_node_try_load_from_on_disk_cache!(dep_node, tcx) + macro_rules! try_load_from_on_disk_cache { + ($($name:ident,)*) => { + match dep_node.kind { + $(DepKind::$name => { + if <query_keys::$name<'tcx> as DepNodeParams<TyCtxt<'_>>>::can_reconstruct_query_key() { + debug_assert!(tcx.dep_graph + .node_color(dep_node) + .map(|c| c.is_green()) + .unwrap_or(false)); + + let key = <query_keys::$name<'tcx> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node).unwrap(); + if queries::$name::cache_on_disk(tcx, &key, None) { + let _ = tcx.$name(key); + } + } + })* + + _ => (), + } + } + } + + rustc_cached_queries!(try_load_from_on_disk_cache!); } mod sealed { diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 27bf22dac75..d038695283c 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -40,7 +40,8 @@ impl QueryContext for TyCtxt<'tcx> { fn try_collect_active_jobs( &self, - ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>> { + ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self::DepKind, Self::Query>>> + { self.queries.try_collect_active_jobs() } @@ -241,25 +242,15 @@ macro_rules! hash_result { }; } -macro_rules! define_queries { - (<$tcx:tt> $($category:tt { - $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($($K:tt)*) -> $V:ty,)* - },)*) => { - define_queries_inner! { <$tcx> - $($( $(#[$attr])* category<$category> [$($modifiers)*] fn $name: $node($($K)*) -> $V,)*)* - } - } -} - macro_rules! query_helper_param_ty { (DefId) => { impl IntoQueryParam<DefId> }; ($K:ty) => { $K }; } -macro_rules! define_queries_inner { +macro_rules! define_queries { (<$tcx:tt> - $($(#[$attr:meta])* category<$category:tt> - [$($modifiers:tt)*] fn $name:ident: $node:ident($($K:tt)*) -> $V:ty,)*) => { + $($(#[$attr:meta])* + [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { use std::mem; use crate::{ @@ -267,7 +258,6 @@ macro_rules! define_queries_inner { rustc_data_structures::stable_hasher::StableHasher, ich::StableHashingContext }; - use rustc_data_structures::profiling::ProfileCategory; define_queries_struct! { tcx: $tcx, @@ -353,7 +343,7 @@ macro_rules! define_queries_inner { $(pub type $name<$tcx> = $V;)* } - $(impl<$tcx> QueryConfig<TyCtxt<$tcx>> for queries::$name<$tcx> { + $(impl<$tcx> QueryConfig for queries::$name<$tcx> { type Key = $($K)*; type Value = $V; type Stored = < @@ -361,18 +351,17 @@ macro_rules! define_queries_inner { as QueryStorage >::Stored; const NAME: &'static str = stringify!($name); - const CATEGORY: ProfileCategory = $category; } impl<$tcx> QueryAccessors<TyCtxt<$tcx>> for queries::$name<$tcx> { const ANON: bool = is_anon!([$($modifiers)*]); const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]); - const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$node; + const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name; type Cache = query_storage!([$($modifiers)*][$($K)*, $V]); #[inline(always)] - fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<TyCtxt<$tcx>, Self::Cache> { + fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, <TyCtxt<$tcx> as QueryContext>::Query, Self::Cache> { &tcx.queries.$name } @@ -454,7 +443,7 @@ macro_rules! define_queries_inner { #[inline(always)] #[must_use] pub fn $name(self, key: query_helper_param_ty!($($K)*)) - -> <queries::$name<$tcx> as QueryConfig<TyCtxt<$tcx>>>::Stored + -> <queries::$name<$tcx> as QueryConfig>::Stored { self.at(DUMMY_SP).$name(key.into_query_param()) })* @@ -493,7 +482,7 @@ macro_rules! define_queries_inner { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) - -> <queries::$name<$tcx> as QueryConfig<TyCtxt<$tcx>>>::Stored + -> <queries::$name<$tcx> as QueryConfig>::Stored { get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param()) })* @@ -527,7 +516,8 @@ macro_rules! define_queries_struct { fallback_extern_providers: Box<Providers>, $($(#[$attr])* $name: QueryState< - TyCtxt<$tcx>, + crate::dep_graph::DepKind, + <TyCtxt<$tcx> as QueryContext>::Query, <queries::$name<$tcx> as QueryAccessors<TyCtxt<'tcx>>>::Cache, >,)* } @@ -548,7 +538,7 @@ macro_rules! define_queries_struct { pub(crate) fn try_collect_active_jobs( &self - ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<TyCtxt<'tcx>>>> { + ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<crate::dep_graph::DepKind, <TyCtxt<$tcx> as QueryContext>::Query>>> { let mut jobs = FxHashMap::default(); $( diff --git a/compiler/rustc_middle/src/ty/query/profiling_support.rs b/compiler/rustc_middle/src/ty/query/profiling_support.rs index 4e8db3194bd..cbcecb88491 100644 --- a/compiler/rustc_middle/src/ty/query/profiling_support.rs +++ b/compiler/rustc_middle/src/ty/query/profiling_support.rs @@ -5,8 +5,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::SelfProfiler; use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; use rustc_hir::definitions::DefPathData; -use rustc_query_system::query::QueryCache; -use rustc_query_system::query::QueryState; +use rustc_query_system::query::{QueryCache, QueryContext, QueryState}; use std::fmt::Debug; use std::io::Write; @@ -231,7 +230,7 @@ where pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( tcx: TyCtxt<'tcx>, query_name: &'static str, - query_state: &QueryState<TyCtxt<'tcx>, C>, + query_state: &QueryState<crate::dep_graph::DepKind, <TyCtxt<'tcx> as QueryContext>::Query, C>, string_cache: &mut QueryKeyStringCache, ) where C: QueryCache, diff --git a/compiler/rustc_middle/src/ty/query/stats.rs b/compiler/rustc_middle/src/ty/query/stats.rs index b496bf839ab..e0b44ce23c9 100644 --- a/compiler/rustc_middle/src/ty/query/stats.rs +++ b/compiler/rustc_middle/src/ty/query/stats.rs @@ -1,11 +1,10 @@ use crate::ty::query::queries; use crate::ty::TyCtxt; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -use rustc_query_system::query::QueryCache; -use rustc_query_system::query::QueryState; -use rustc_query_system::query::{QueryAccessors, QueryContext}; +use rustc_query_system::query::{QueryAccessors, QueryCache, QueryContext, QueryState}; use std::any::type_name; +use std::hash::Hash; use std::mem; #[cfg(debug_assertions)] use std::sync::atomic::Ordering; @@ -38,10 +37,12 @@ struct QueryStats { local_def_id_keys: Option<usize>, } -fn stats<CTX: QueryContext, C: QueryCache>( - name: &'static str, - map: &QueryState<CTX, C>, -) -> QueryStats { +fn stats<D, Q, C>(name: &'static str, map: &QueryState<D, Q, C>) -> QueryStats +where + D: Copy + Clone + Eq + Hash, + Q: Clone, + C: QueryCache, +{ let mut stats = QueryStats { name, #[cfg(debug_assertions)] @@ -119,21 +120,22 @@ pub fn print_stats(tcx: TyCtxt<'_>) { } macro_rules! print_stats { - (<$tcx:tt> $($category:tt { - $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident: $node:ident($K:ty) -> $V:ty,)* - },)*) => { + (<$tcx:tt> + $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)* + ) => { fn query_stats(tcx: TyCtxt<'_>) -> Vec<QueryStats> { let mut queries = Vec::new(); - $($( + $( queries.push(stats::< - TyCtxt<'_>, + crate::dep_graph::DepKind, + <TyCtxt<'_> as QueryContext>::Query, <queries::$name<'_> as QueryAccessors<TyCtxt<'_>>>::Cache, >( stringify!($name), &tcx.queries.$name, )); - )*)* + )* queries } diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs index c4df0bba726..ef5034e218d 100644 --- a/compiler/rustc_middle/src/ty/relate.rs +++ b/compiler/rustc_middle/src/ty/relate.rs @@ -490,7 +490,7 @@ pub fn super_relate_consts<R: TypeRelation<'tcx>>( let eagerly_eval = |x: &'tcx ty::Const<'tcx>| x.eval(tcx, relation.param_env()).val; // FIXME(eddyb) doesn't look like everything below checks that `a.ty == b.ty`. - // We could probably always assert it early, as `const` generic parameters + // We could probably always assert it early, as const generic parameters // are not allowed to depend on other generic parameters, i.e. are concrete. // (although there could be normalization differences) diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 15803dbd842..431225e2767 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -14,6 +14,7 @@ use rustc_index::vec::{Idx, IndexVec}; use smallvec::SmallVec; use std::fmt; +use std::ops::ControlFlow; use std::rc::Rc; use std::sync::Arc; @@ -332,24 +333,23 @@ CloneTypeFoldableAndLiftImpls! { // FIXME(eddyb) replace all the uses of `Option::map` with `?`. impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>> Lift<'tcx> for (A, B) { type Lifted = (A::Lifted, B::Lifted); - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.0).and_then(|a| tcx.lift(&self.1).map(|b| (a, b))) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + Some((tcx.lift(self.0)?, tcx.lift(self.1)?)) } } impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>, C: Lift<'tcx>> Lift<'tcx> for (A, B, C) { type Lifted = (A::Lifted, B::Lifted, C::Lifted); - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.0) - .and_then(|a| tcx.lift(&self.1).and_then(|b| tcx.lift(&self.2).map(|c| (a, b, c)))) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + Some((tcx.lift(self.0)?, tcx.lift(self.1)?, tcx.lift(self.2)?)) } } impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Option<T> { type Lifted = Option<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { - Some(ref x) => tcx.lift(x).map(Some), + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { + Some(x) => tcx.lift(x).map(Some), None => Some(None), } } @@ -357,89 +357,72 @@ impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Option<T> { impl<'tcx, T: Lift<'tcx>, E: Lift<'tcx>> Lift<'tcx> for Result<T, E> { type Lifted = Result<T::Lifted, E::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { - Ok(ref x) => tcx.lift(x).map(Ok), - Err(ref e) => tcx.lift(e).map(Err), + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { + Ok(x) => tcx.lift(x).map(Ok), + Err(e) => tcx.lift(e).map(Err), } } } impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Box<T> { type Lifted = Box<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&**self).map(Box::new) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(*self).map(Box::new) } } -impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Rc<T> { +impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Rc<T> { type Lifted = Rc<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&**self).map(Rc::new) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.as_ref().clone()).map(Rc::new) } } -impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Arc<T> { +impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Arc<T> { type Lifted = Arc<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&**self).map(Arc::new) - } -} - -impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for [T] { - type Lifted = Vec<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - // type annotation needed to inform `projection_must_outlive` - let mut result: Vec<<T as Lift<'tcx>>::Lifted> = Vec::with_capacity(self.len()); - for x in self { - if let Some(value) = tcx.lift(x) { - result.push(value); - } else { - return None; - } - } - Some(result) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.as_ref().clone()).map(Arc::new) } } - impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Vec<T> { type Lifted = Vec<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self[..]) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + self.into_iter().map(|v| tcx.lift(v)).collect() } } impl<'tcx, I: Idx, T: Lift<'tcx>> Lift<'tcx> for IndexVec<I, T> { type Lifted = IndexVec<I, T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - self.iter().map(|e| tcx.lift(e)).collect() + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + self.into_iter().map(|e| tcx.lift(e)).collect() } } impl<'a, 'tcx> Lift<'tcx> for ty::TraitRef<'a> { type Lifted = ty::TraitRef<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.substs).map(|substs| ty::TraitRef { def_id: self.def_id, substs }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.substs).map(|substs| ty::TraitRef { def_id: self.def_id, substs }) } } impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialTraitRef<'a> { type Lifted = ty::ExistentialTraitRef<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.substs).map(|substs| ty::ExistentialTraitRef { def_id: self.def_id, substs }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.substs).map(|substs| ty::ExistentialTraitRef { def_id: self.def_id, substs }) } } impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialPredicate<'a> { type Lifted = ty::ExistentialPredicate<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { match self { ty::ExistentialPredicate::Trait(x) => tcx.lift(x).map(ty::ExistentialPredicate::Trait), ty::ExistentialPredicate::Projection(x) => { tcx.lift(x).map(ty::ExistentialPredicate::Projection) } ty::ExistentialPredicate::AutoTrait(def_id) => { - Some(ty::ExistentialPredicate::AutoTrait(*def_id)) + Some(ty::ExistentialPredicate::AutoTrait(def_id)) } } } @@ -447,15 +430,15 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialPredicate<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::TraitPredicate<'a> { type Lifted = ty::TraitPredicate<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<ty::TraitPredicate<'tcx>> { - tcx.lift(&self.trait_ref).map(|trait_ref| ty::TraitPredicate { trait_ref }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::TraitPredicate<'tcx>> { + tcx.lift(self.trait_ref).map(|trait_ref| ty::TraitPredicate { trait_ref }) } } impl<'a, 'tcx> Lift<'tcx> for ty::SubtypePredicate<'a> { type Lifted = ty::SubtypePredicate<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<ty::SubtypePredicate<'tcx>> { - tcx.lift(&(self.a, self.b)).map(|(a, b)| ty::SubtypePredicate { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::SubtypePredicate<'tcx>> { + tcx.lift((self.a, self.b)).map(|(a, b)| ty::SubtypePredicate { a_is_expected: self.a_is_expected, a, b, @@ -465,33 +448,33 @@ impl<'a, 'tcx> Lift<'tcx> for ty::SubtypePredicate<'a> { impl<'tcx, A: Copy + Lift<'tcx>, B: Copy + Lift<'tcx>> Lift<'tcx> for ty::OutlivesPredicate<A, B> { type Lifted = ty::OutlivesPredicate<A::Lifted, B::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&(self.0, self.1)).map(|(a, b)| ty::OutlivesPredicate(a, b)) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift((self.0, self.1)).map(|(a, b)| ty::OutlivesPredicate(a, b)) } } impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionTy<'a> { type Lifted = ty::ProjectionTy<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionTy<'tcx>> { - tcx.lift(&self.substs) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionTy<'tcx>> { + tcx.lift(self.substs) .map(|substs| ty::ProjectionTy { item_def_id: self.item_def_id, substs }) } } impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionPredicate<'a> { type Lifted = ty::ProjectionPredicate<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionPredicate<'tcx>> { - tcx.lift(&(self.projection_ty, self.ty)) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionPredicate<'tcx>> { + tcx.lift((self.projection_ty, self.ty)) .map(|(projection_ty, ty)| ty::ProjectionPredicate { projection_ty, ty }) } } impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialProjection<'a> { type Lifted = ty::ExistentialProjection<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.substs).map(|substs| ty::ExistentialProjection { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.substs).map(|substs| ty::ExistentialProjection { substs, - ty: tcx.lift(&self.ty).expect("type must lift when substs do"), + ty: tcx.lift(self.ty).expect("type must lift when substs do"), item_def_id: self.item_def_id, }) } @@ -499,7 +482,7 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialProjection<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::PredicateKind<'a> { type Lifted = ty::PredicateKind<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { match self { ty::PredicateKind::ForAll(binder) => tcx.lift(binder).map(ty::PredicateKind::ForAll), ty::PredicateKind::Atom(atom) => tcx.lift(atom).map(ty::PredicateKind::Atom), @@ -509,24 +492,24 @@ impl<'a, 'tcx> Lift<'tcx> for ty::PredicateKind<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::PredicateAtom<'a> { type Lifted = ty::PredicateAtom<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { - ty::PredicateAtom::Trait(ref data, constness) => { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { + ty::PredicateAtom::Trait(data, constness) => { tcx.lift(data).map(|data| ty::PredicateAtom::Trait(data, constness)) } - ty::PredicateAtom::Subtype(ref data) => tcx.lift(data).map(ty::PredicateAtom::Subtype), - ty::PredicateAtom::RegionOutlives(ref data) => { + ty::PredicateAtom::Subtype(data) => tcx.lift(data).map(ty::PredicateAtom::Subtype), + ty::PredicateAtom::RegionOutlives(data) => { tcx.lift(data).map(ty::PredicateAtom::RegionOutlives) } - ty::PredicateAtom::TypeOutlives(ref data) => { + ty::PredicateAtom::TypeOutlives(data) => { tcx.lift(data).map(ty::PredicateAtom::TypeOutlives) } - ty::PredicateAtom::Projection(ref data) => { + ty::PredicateAtom::Projection(data) => { tcx.lift(data).map(ty::PredicateAtom::Projection) } - ty::PredicateAtom::WellFormed(ty) => tcx.lift(&ty).map(ty::PredicateAtom::WellFormed), + ty::PredicateAtom::WellFormed(ty) => tcx.lift(ty).map(ty::PredicateAtom::WellFormed), ty::PredicateAtom::ClosureKind(closure_def_id, closure_substs, kind) => { - tcx.lift(&closure_substs).map(|closure_substs| { + tcx.lift(closure_substs).map(|closure_substs| { ty::PredicateAtom::ClosureKind(closure_def_id, closure_substs, kind) }) } @@ -534,13 +517,13 @@ impl<'a, 'tcx> Lift<'tcx> for ty::PredicateAtom<'a> { Some(ty::PredicateAtom::ObjectSafe(trait_def_id)) } ty::PredicateAtom::ConstEvaluatable(def_id, substs) => { - tcx.lift(&substs).map(|substs| ty::PredicateAtom::ConstEvaluatable(def_id, substs)) + tcx.lift(substs).map(|substs| ty::PredicateAtom::ConstEvaluatable(def_id, substs)) } ty::PredicateAtom::ConstEquate(c1, c2) => { - tcx.lift(&(c1, c2)).map(|(c1, c2)| ty::PredicateAtom::ConstEquate(c1, c2)) + tcx.lift((c1, c2)).map(|(c1, c2)| ty::PredicateAtom::ConstEquate(c1, c2)) } ty::PredicateAtom::TypeWellFormedFromEnv(ty) => { - tcx.lift(&ty).map(ty::PredicateAtom::TypeWellFormedFromEnv) + tcx.lift(ty).map(ty::PredicateAtom::TypeWellFormedFromEnv) } } } @@ -548,61 +531,62 @@ impl<'a, 'tcx> Lift<'tcx> for ty::PredicateAtom<'a> { impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::Binder<T> { type Lifted = ty::Binder<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(self.as_ref().skip_binder()).map(|v| self.rebind(v)) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + self.map_bound(|v| tcx.lift(v)).transpose() } } impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> { type Lifted = ty::ParamEnv<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.caller_bounds()) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.caller_bounds()) .map(|caller_bounds| ty::ParamEnv::new(caller_bounds, self.reveal())) } } impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::ParamEnvAnd<'a, T> { type Lifted = ty::ParamEnvAnd<'tcx, T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.param_env).and_then(|param_env| { - tcx.lift(&self.value).map(|value| ty::ParamEnvAnd { param_env, value }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.param_env).and_then(|param_env| { + tcx.lift(self.value).map(|value| ty::ParamEnvAnd { param_env, value }) }) } } impl<'a, 'tcx> Lift<'tcx> for ty::ClosureSubsts<'a> { type Lifted = ty::ClosureSubsts<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.substs).map(|substs| ty::ClosureSubsts { substs }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.substs).map(|substs| ty::ClosureSubsts { substs }) } } impl<'a, 'tcx> Lift<'tcx> for ty::GeneratorSubsts<'a> { type Lifted = ty::GeneratorSubsts<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.substs).map(|substs| ty::GeneratorSubsts { substs }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.substs).map(|substs| ty::GeneratorSubsts { substs }) } } impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjustment<'a> { type Lifted = ty::adjustment::Adjustment<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.kind).and_then(|kind| { - tcx.lift(&self.target).map(|target| ty::adjustment::Adjustment { kind, target }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + let ty::adjustment::Adjustment { kind, target } = self; + tcx.lift(kind).and_then(|kind| { + tcx.lift(target).map(|target| ty::adjustment::Adjustment { kind, target }) }) } } impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjust<'a> { type Lifted = ty::adjustment::Adjust<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { ty::adjustment::Adjust::NeverToAny => Some(ty::adjustment::Adjust::NeverToAny), ty::adjustment::Adjust::Pointer(ptr) => Some(ty::adjustment::Adjust::Pointer(ptr)), - ty::adjustment::Adjust::Deref(ref overloaded) => { + ty::adjustment::Adjust::Deref(overloaded) => { tcx.lift(overloaded).map(ty::adjustment::Adjust::Deref) } - ty::adjustment::Adjust::Borrow(ref autoref) => { + ty::adjustment::Adjust::Borrow(autoref) => { tcx.lift(autoref).map(ty::adjustment::Adjust::Borrow) } } @@ -611,8 +595,8 @@ impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjust<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::OverloadedDeref<'a> { type Lifted = ty::adjustment::OverloadedDeref<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.region).map(|region| ty::adjustment::OverloadedDeref { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.region).map(|region| ty::adjustment::OverloadedDeref { region, mutbl: self.mutbl, span: self.span, @@ -622,10 +606,10 @@ impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::OverloadedDeref<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::AutoBorrow<'a> { type Lifted = ty::adjustment::AutoBorrow<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { ty::adjustment::AutoBorrow::Ref(r, m) => { - tcx.lift(&r).map(|r| ty::adjustment::AutoBorrow::Ref(r, m)) + tcx.lift(r).map(|r| ty::adjustment::AutoBorrow::Ref(r, m)) } ty::adjustment::AutoBorrow::RawPtr(m) => Some(ty::adjustment::AutoBorrow::RawPtr(m)), } @@ -634,16 +618,16 @@ impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::AutoBorrow<'a> { impl<'a, 'tcx> Lift<'tcx> for ty::GenSig<'a> { type Lifted = ty::GenSig<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&(self.resume_ty, self.yield_ty, self.return_ty)) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift((self.resume_ty, self.yield_ty, self.return_ty)) .map(|(resume_ty, yield_ty, return_ty)| ty::GenSig { resume_ty, yield_ty, return_ty }) } } impl<'a, 'tcx> Lift<'tcx> for ty::FnSig<'a> { type Lifted = ty::FnSig<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.inputs_and_output).map(|x| ty::FnSig { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + tcx.lift(self.inputs_and_output).map(|x| ty::FnSig { inputs_and_output: x, c_variadic: self.c_variadic, unsafety: self.unsafety, @@ -654,19 +638,20 @@ impl<'a, 'tcx> Lift<'tcx> for ty::FnSig<'a> { impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::error::ExpectedFound<T> { type Lifted = ty::error::ExpectedFound<T::Lifted>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - tcx.lift(&self.expected).and_then(|expected| { - tcx.lift(&self.found).map(|found| ty::error::ExpectedFound { expected, found }) + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + let ty::error::ExpectedFound { expected, found } = self; + tcx.lift(expected).and_then(|expected| { + tcx.lift(found).map(|found| ty::error::ExpectedFound { expected, found }) }) } } impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> { type Lifted = ty::error::TypeError<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { use crate::ty::error::TypeError::*; - Some(match *self { + Some(match self { Mismatch => Mismatch, UnsafetyMismatch(x) => UnsafetyMismatch(x), AbiMismatch(x) => AbiMismatch(x), @@ -675,51 +660,51 @@ impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> { FixedArraySize(x) => FixedArraySize(x), ArgCount => ArgCount, RegionsDoesNotOutlive(a, b) => { - return tcx.lift(&(a, b)).map(|(a, b)| RegionsDoesNotOutlive(a, b)); + return tcx.lift((a, b)).map(|(a, b)| RegionsDoesNotOutlive(a, b)); } RegionsInsufficientlyPolymorphic(a, b) => { - return tcx.lift(&b).map(|b| RegionsInsufficientlyPolymorphic(a, b)); + return tcx.lift(b).map(|b| RegionsInsufficientlyPolymorphic(a, b)); } RegionsOverlyPolymorphic(a, b) => { - return tcx.lift(&b).map(|b| RegionsOverlyPolymorphic(a, b)); + return tcx.lift(b).map(|b| RegionsOverlyPolymorphic(a, b)); } RegionsPlaceholderMismatch => RegionsPlaceholderMismatch, IntMismatch(x) => IntMismatch(x), FloatMismatch(x) => FloatMismatch(x), Traits(x) => Traits(x), VariadicMismatch(x) => VariadicMismatch(x), - CyclicTy(t) => return tcx.lift(&t).map(|t| CyclicTy(t)), - CyclicConst(ct) => return tcx.lift(&ct).map(|ct| CyclicConst(ct)), + CyclicTy(t) => return tcx.lift(t).map(|t| CyclicTy(t)), + CyclicConst(ct) => return tcx.lift(ct).map(|ct| CyclicConst(ct)), ProjectionMismatched(x) => ProjectionMismatched(x), - Sorts(ref x) => return tcx.lift(x).map(Sorts), - ExistentialMismatch(ref x) => return tcx.lift(x).map(ExistentialMismatch), - ConstMismatch(ref x) => return tcx.lift(x).map(ConstMismatch), + Sorts(x) => return tcx.lift(x).map(Sorts), + ExistentialMismatch(x) => return tcx.lift(x).map(ExistentialMismatch), + ConstMismatch(x) => return tcx.lift(x).map(ConstMismatch), IntrinsicCast => IntrinsicCast, - TargetFeatureCast(ref x) => TargetFeatureCast(*x), - ObjectUnsafeCoercion(ref x) => return tcx.lift(x).map(ObjectUnsafeCoercion), + TargetFeatureCast(x) => TargetFeatureCast(x), + ObjectUnsafeCoercion(x) => return tcx.lift(x).map(ObjectUnsafeCoercion), }) } } impl<'a, 'tcx> Lift<'tcx> for ty::InstanceDef<'a> { type Lifted = ty::InstanceDef<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { - match *self { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + match self { ty::InstanceDef::Item(def_id) => Some(ty::InstanceDef::Item(def_id)), ty::InstanceDef::VtableShim(def_id) => Some(ty::InstanceDef::VtableShim(def_id)), ty::InstanceDef::ReifyShim(def_id) => Some(ty::InstanceDef::ReifyShim(def_id)), ty::InstanceDef::Intrinsic(def_id) => Some(ty::InstanceDef::Intrinsic(def_id)), - ty::InstanceDef::FnPtrShim(def_id, ref ty) => { + ty::InstanceDef::FnPtrShim(def_id, ty) => { Some(ty::InstanceDef::FnPtrShim(def_id, tcx.lift(ty)?)) } ty::InstanceDef::Virtual(def_id, n) => Some(ty::InstanceDef::Virtual(def_id, n)), ty::InstanceDef::ClosureOnceShim { call_once } => { Some(ty::InstanceDef::ClosureOnceShim { call_once }) } - ty::InstanceDef::DropGlue(def_id, ref ty) => { + ty::InstanceDef::DropGlue(def_id, ty) => { Some(ty::InstanceDef::DropGlue(def_id, tcx.lift(ty)?)) } - ty::InstanceDef::CloneShim(def_id, ref ty) => { + ty::InstanceDef::CloneShim(def_id, ty) => { Some(ty::InstanceDef::CloneShim(def_id, tcx.lift(ty)?)) } } @@ -743,8 +728,8 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::AdtDef { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } @@ -753,8 +738,9 @@ impl<'tcx, T: TypeFoldable<'tcx>, U: TypeFoldable<'tcx>> TypeFoldable<'tcx> for (self.0.fold_with(folder), self.1.fold_with(folder)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.0.visit_with(visitor) || self.1.visit_with(visitor) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.0.visit_with(visitor)?; + self.1.visit_with(visitor) } } @@ -765,8 +751,10 @@ impl<'tcx, A: TypeFoldable<'tcx>, B: TypeFoldable<'tcx>, C: TypeFoldable<'tcx>> (self.0.fold_with(folder), self.1.fold_with(folder), self.2.fold_with(folder)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.0.visit_with(visitor) || self.1.visit_with(visitor) || self.2.visit_with(visitor) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.0.visit_with(visitor)?; + self.1.visit_with(visitor)?; + self.2.visit_with(visitor) } } @@ -789,7 +777,7 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Rc<T> { Rc::new((**self).fold_with(folder)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { (**self).visit_with(visitor) } } @@ -799,7 +787,7 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Arc<T> { Arc::new((**self).fold_with(folder)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { (**self).visit_with(visitor) } } @@ -810,7 +798,7 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<T> { box content } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { (**self).visit_with(visitor) } } @@ -820,8 +808,8 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Vec<T> { self.iter().map(|t| t.fold_with(folder)).collect() } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -830,8 +818,8 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<[T]> { self.iter().map(|t| t.fold_with(folder)).collect::<Vec<_>>().into_boxed_slice() } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -844,11 +832,11 @@ impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for ty::Binder<T> { folder.fold_binder(self) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { self.as_ref().skip_binder().visit_with(visitor) } - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { visitor.visit_binder(self) } } @@ -858,8 +846,8 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::ExistentialPredicate<'tcx>> fold_list(*self, folder, |tcx, v| tcx.intern_existential_predicates(v)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|p| p.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|p| p.visit_with(visitor)) } } @@ -868,8 +856,8 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> { fold_list(*self, folder, |tcx, v| tcx.intern_type_list(v)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -878,8 +866,8 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ProjectionKind> { fold_list(*self, folder, |tcx, v| tcx.intern_projs(v)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -904,20 +892,24 @@ impl<'tcx> TypeFoldable<'tcx> for ty::instance::Instance<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { use crate::ty::InstanceDef::*; - self.substs.visit_with(visitor) - || match self.def { - Item(def) => def.visit_with(visitor), - VtableShim(did) | ReifyShim(did) | Intrinsic(did) | Virtual(did, _) => { - did.visit_with(visitor) - } - FnPtrShim(did, ty) | CloneShim(did, ty) => { - did.visit_with(visitor) || ty.visit_with(visitor) - } - DropGlue(did, ty) => did.visit_with(visitor) || ty.visit_with(visitor), - ClosureOnceShim { call_once } => call_once.visit_with(visitor), + self.substs.visit_with(visitor)?; + match self.def { + Item(def) => def.visit_with(visitor), + VtableShim(did) | ReifyShim(did) | Intrinsic(did) | Virtual(did, _) => { + did.visit_with(visitor) + } + FnPtrShim(did, ty) | CloneShim(did, ty) => { + did.visit_with(visitor)?; + ty.visit_with(visitor) } + DropGlue(did, ty) => { + did.visit_with(visitor)?; + ty.visit_with(visitor) + } + ClosureOnceShim { call_once } => call_once.visit_with(visitor), + } } } @@ -926,7 +918,7 @@ impl<'tcx> TypeFoldable<'tcx> for interpret::GlobalId<'tcx> { Self { instance: self.instance.fold_with(folder), promoted: self.promoted } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { self.instance.visit_with(visitor) } } @@ -975,19 +967,26 @@ impl<'tcx> TypeFoldable<'tcx> for Ty<'tcx> { folder.fold_ty(*self) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { match self.kind() { ty::RawPtr(ref tm) => tm.visit_with(visitor), - ty::Array(typ, sz) => typ.visit_with(visitor) || sz.visit_with(visitor), + ty::Array(typ, sz) => { + typ.visit_with(visitor)?; + sz.visit_with(visitor) + } ty::Slice(typ) => typ.visit_with(visitor), ty::Adt(_, substs) => substs.visit_with(visitor), ty::Dynamic(ref trait_ty, ref reg) => { - trait_ty.visit_with(visitor) || reg.visit_with(visitor) + trait_ty.visit_with(visitor)?; + reg.visit_with(visitor) } ty::Tuple(ts) => ts.visit_with(visitor), ty::FnDef(_, substs) => substs.visit_with(visitor), ty::FnPtr(ref f) => f.visit_with(visitor), - ty::Ref(r, ty, _) => r.visit_with(visitor) || ty.visit_with(visitor), + ty::Ref(r, ty, _) => { + r.visit_with(visitor)?; + ty.visit_with(visitor) + } ty::Generator(_did, ref substs, _) => substs.visit_with(visitor), ty::GeneratorWitness(ref types) => types.visit_with(visitor), ty::Closure(_did, ref substs) => substs.visit_with(visitor), @@ -1006,11 +1005,11 @@ impl<'tcx> TypeFoldable<'tcx> for Ty<'tcx> { | ty::Placeholder(..) | ty::Param(..) | ty::Never - | ty::Foreign(..) => false, + | ty::Foreign(..) => ControlFlow::CONTINUE, } } - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { visitor.visit_ty(self) } } @@ -1024,11 +1023,11 @@ impl<'tcx> TypeFoldable<'tcx> for ty::Region<'tcx> { folder.fold_region(*self) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { visitor.visit_region(*self) } } @@ -1039,11 +1038,11 @@ impl<'tcx> TypeFoldable<'tcx> for ty::Predicate<'tcx> { folder.tcx().reuse_or_mk_predicate(*self, new) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { ty::PredicateKind::super_visit_with(&self.inner.kind, visitor) } - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { visitor.visit_predicate(*self) } @@ -1056,23 +1055,13 @@ impl<'tcx> TypeFoldable<'tcx> for ty::Predicate<'tcx> { } } -pub(super) trait PredicateVisitor<'tcx>: TypeVisitor<'tcx> { - fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool; -} - -impl<T: TypeVisitor<'tcx>> PredicateVisitor<'tcx> for T { - default fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool { - predicate.super_visit_with(self) - } -} - impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> { fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self { fold_list(*self, folder, |tcx, v| tcx.intern_predicates(v)) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|p| p.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|p| p.visit_with(visitor)) } } @@ -1081,8 +1070,8 @@ impl<'tcx, T: TypeFoldable<'tcx>, I: Idx> TypeFoldable<'tcx> for IndexVec<I, T> self.iter().map(|x| x.fold_with(folder)).collect() } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } @@ -1101,11 +1090,12 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::Const<'tcx> { folder.fold_const(*self) } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.ty.visit_with(visitor) || self.val.visit_with(visitor) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.ty.visit_with(visitor)?; + self.val.visit_with(visitor) } - fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { visitor.visit_const(self) } } @@ -1125,7 +1115,7 @@ impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { match *self { ty::ConstKind::Infer(ic) => ic.visit_with(visitor), ty::ConstKind::Param(p) => p.visit_with(visitor), @@ -1133,7 +1123,7 @@ impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> { ty::ConstKind::Value(_) | ty::ConstKind::Bound(..) | ty::ConstKind::Placeholder(_) - | ty::ConstKind::Error(_) => false, + | ty::ConstKind::Error(_) => ControlFlow::CONTINUE, } } } @@ -1143,8 +1133,8 @@ impl<'tcx> TypeFoldable<'tcx> for InferConst<'tcx> { *self } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> bool { - false + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<()> { + ControlFlow::CONTINUE } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 633b9fe9acd..384d08f8348 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -210,6 +210,18 @@ impl TyKind<'tcx> { _ => false, } } + + /// Get the article ("a" or "an") to use with this type. + pub fn article(&self) -> &'static str { + match self { + Int(_) | Float(_) | Array(_, _) => "an", + Adt(def, _) if def.is_enum() => "an", + // This should never happen, but ICEing and causing the user's code + // to not compile felt too harsh. + Error(_) => "a", + _ => "a", + } + } } // `TyKind` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -376,9 +388,19 @@ impl<'tcx> ClosureSubsts<'tcx> { self.split().parent_substs } + /// Returns an iterator over the list of types of captured paths by the closure. + /// In case there was a type error in figuring out the types of the captured path, an + /// empty iterator is returned. #[inline] pub fn upvar_tys(self) -> impl Iterator<Item = Ty<'tcx>> + 'tcx { - self.tupled_upvars_ty().tuple_fields() + match self.tupled_upvars_ty().kind() { + TyKind::Error(_) => None, + TyKind::Tuple(..) => Some(self.tupled_upvars_ty().tuple_fields()), + TyKind::Infer(_) => bug!("upvar_tys called before capture types are inferred"), + ty => bug!("Unexpected representation of upvar types tuple {:?}", ty), + } + .into_iter() + .flatten() } /// Returns the tuple type representing the upvars for this closure. @@ -503,9 +525,19 @@ impl<'tcx> GeneratorSubsts<'tcx> { self.split().witness.expect_ty() } + /// Returns an iterator over the list of types of captured paths by the generator. + /// In case there was a type error in figuring out the types of the captured path, an + /// empty iterator is returned. #[inline] pub fn upvar_tys(self) -> impl Iterator<Item = Ty<'tcx>> + 'tcx { - self.tupled_upvars_ty().tuple_fields() + match self.tupled_upvars_ty().kind() { + TyKind::Error(_) => None, + TyKind::Tuple(..) => Some(self.tupled_upvars_ty().tuple_fields()), + TyKind::Infer(_) => bug!("upvar_tys called before capture types are inferred"), + ty => bug!("Unexpected representation of upvar types tuple {:?}", ty), + } + .into_iter() + .flatten() } /// Returns the tuple type representing the upvars for this generator. @@ -648,13 +680,24 @@ pub enum UpvarSubsts<'tcx> { } impl<'tcx> UpvarSubsts<'tcx> { + /// Returns an iterator over the list of types of captured paths by the closure/generator. + /// In case there was a type error in figuring out the types of the captured path, an + /// empty iterator is returned. #[inline] pub fn upvar_tys(self) -> impl Iterator<Item = Ty<'tcx>> + 'tcx { - let tupled_upvars_ty = match self { - UpvarSubsts::Closure(substs) => substs.as_closure().split().tupled_upvars_ty, - UpvarSubsts::Generator(substs) => substs.as_generator().split().tupled_upvars_ty, + let tupled_tys = match self { + UpvarSubsts::Closure(substs) => substs.as_closure().tupled_upvars_ty(), + UpvarSubsts::Generator(substs) => substs.as_generator().tupled_upvars_ty(), }; - tupled_upvars_ty.expect_ty().tuple_fields() + + match tupled_tys.kind() { + TyKind::Error(_) => None, + TyKind::Tuple(..) => Some(self.tupled_upvars_ty().tuple_fields()), + TyKind::Infer(_) => bug!("upvar_tys called before capture types are inferred"), + ty => bug!("Unexpected representation of upvar types tuple {:?}", ty), + } + .into_iter() + .flatten() } #[inline] @@ -1822,10 +1865,10 @@ impl<'tcx> TyS<'tcx> { } ty::Array(ty, len) => { match len.try_eval_usize(tcx, ParamEnv::empty()) { + Some(0) | None => false, // If the array is definitely non-empty, it's uninhabited if // the type of its elements is uninhabited. - Some(n) if n != 0 => ty.conservative_is_privately_uninhabited(tcx), - _ => false, + Some(1..) => ty.conservative_is_privately_uninhabited(tcx), } } ty::Ref(..) => { diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs index 7d96adb7c8b..07f775cf8b1 100644 --- a/compiler/rustc_middle/src/ty/subst.rs +++ b/compiler/rustc_middle/src/ty/subst.rs @@ -17,6 +17,7 @@ use std::fmt; use std::marker::PhantomData; use std::mem; use std::num::NonZeroUsize; +use std::ops::ControlFlow; /// An entity in the Rust type system, which can be one of /// several kinds (types, lifetimes, and consts). @@ -141,11 +142,11 @@ impl<'tcx> GenericArg<'tcx> { impl<'a, 'tcx> Lift<'tcx> for GenericArg<'a> { type Lifted = GenericArg<'tcx>; - fn lift_to_tcx(&self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { + fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> { match self.unpack() { - GenericArgKind::Lifetime(lt) => tcx.lift(<).map(|lt| lt.into()), - GenericArgKind::Type(ty) => tcx.lift(&ty).map(|ty| ty.into()), - GenericArgKind::Const(ct) => tcx.lift(&ct).map(|ct| ct.into()), + GenericArgKind::Lifetime(lt) => tcx.lift(lt).map(|lt| lt.into()), + GenericArgKind::Type(ty) => tcx.lift(ty).map(|ty| ty.into()), + GenericArgKind::Const(ct) => tcx.lift(ct).map(|ct| ct.into()), } } } @@ -159,7 +160,7 @@ impl<'tcx> TypeFoldable<'tcx> for GenericArg<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { match self.unpack() { GenericArgKind::Lifetime(lt) => lt.visit_with(visitor), GenericArgKind::Type(ty) => ty.visit_with(visitor), @@ -391,8 +392,8 @@ impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> { } } - fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool { - self.iter().any(|t| t.visit_with(visitor)) + fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<()> { + self.iter().try_for_each(|t| t.visit_with(visitor)) } } diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index b0f0f0ba57f..4a20e1c32f9 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -529,8 +529,12 @@ impl<'tcx> TyCtxt<'tcx> { // Make sure that any constants in the static's type are evaluated. let static_ty = self.normalize_erasing_regions(ty::ParamEnv::empty(), self.type_of(def_id)); + // Make sure that accesses to unsafe statics end up using raw pointers. + // For thread-locals, this needs to be kept in sync with `Rvalue::ty`. if self.is_mutable_static(def_id) { self.mk_mut_ptr(static_ty) + } else if self.is_foreign_item(def_id) { + self.mk_imm_ptr(static_ty) } else { self.mk_imm_ref(self.lifetimes.re_erased, static_ty) } diff --git a/compiler/rustc_mir/src/borrow_check/mod.rs b/compiler/rustc_mir/src/borrow_check/mod.rs index 4b7af271bae..de54c5582e0 100644 --- a/compiler/rustc_mir/src/borrow_check/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/mod.rs @@ -674,29 +674,16 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc TerminatorKind::SwitchInt { ref discr, switch_ty: _, targets: _ } => { self.consume_operand(loc, (discr, span), flow_state); } - TerminatorKind::Drop { place: ref drop_place, target: _, unwind: _ } => { - let tcx = self.infcx.tcx; - - // Compute the type with accurate region information. - let drop_place_ty = drop_place.ty(self.body, self.infcx.tcx); - - // Erase the regions. - let drop_place_ty = self.infcx.tcx.erase_regions(&drop_place_ty).ty; - - // "Lift" into the tcx -- once regions are erased, this type should be in the - // global arenas; this "lift" operation basically just asserts that is true, but - // that is useful later. - tcx.lift(&drop_place_ty).unwrap(); - + TerminatorKind::Drop { place, target: _, unwind: _ } => { debug!( "visit_terminator_drop \ - loc: {:?} term: {:?} drop_place: {:?} drop_place_ty: {:?} span: {:?}", - loc, term, drop_place, drop_place_ty, span + loc: {:?} term: {:?} place: {:?} span: {:?}", + loc, term, place, span ); self.access_place( loc, - (*drop_place, span), + (place, span), (AccessDepth::Drop, Write(WriteKind::StorageDeadOrDrop)), LocalMutationIsAllowed::Yes, flow_state, diff --git a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs index 4fc1c570e46..409399094e8 100644 --- a/compiler/rustc_mir/src/borrow_check/type_check/mod.rs +++ b/compiler/rustc_mir/src/borrow_check/type_check/mod.rs @@ -974,6 +974,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { checker } + fn unsized_feature_enabled(&self) -> bool { + let features = self.tcx().features(); + features.unsized_locals || features.unsized_fn_params + } + /// Equate the inferred type and the annotated type for user type annotations fn check_user_type_annotations(&mut self) { debug!( @@ -1456,7 +1461,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } self.check_rvalue(body, rv, location); - if !self.tcx().features().unsized_locals { + if !self.unsized_feature_enabled() { let trait_ref = ty::TraitRef { def_id: tcx.require_lang_item(LangItem::Sized, Some(self.last_span)), substs: tcx.mk_substs_trait(place_ty, &[]), @@ -1717,9 +1722,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { ); } - // When `#![feature(unsized_locals)]` is not enabled, + // When `unsized_fn_params` and `unsized_locals` are both not enabled, // this check is done at `check_local`. - if self.tcx().features().unsized_locals { + if self.unsized_feature_enabled() { let span = term.source_info.span; self.ensure_place_sized(dest_ty, span); } @@ -1880,9 +1885,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { LocalKind::Var | LocalKind::Temp => {} } - // When `#![feature(unsized_locals)]` is enabled, only function calls + // When `unsized_fn_params` or `unsized_locals` is enabled, only function calls // and nullary ops are checked in `check_call_dest`. - if !self.tcx().features().unsized_locals { + if !self.unsized_feature_enabled() { let span = local_decl.source_info.span; let ty = local_decl.ty; self.ensure_place_sized(ty, span); @@ -2024,7 +2029,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { Rvalue::NullaryOp(_, ty) => { // Even with unsized locals cannot box an unsized value. - if self.tcx().features().unsized_locals { + if self.unsized_feature_enabled() { let span = body.source_info(location).span; self.ensure_place_sized(ty, span); } diff --git a/compiler/rustc_mir/src/const_eval/eval_queries.rs b/compiler/rustc_mir/src/const_eval/eval_queries.rs index 6ef73b04238..7b9a4ec873d 100644 --- a/compiler/rustc_mir/src/const_eval/eval_queries.rs +++ b/compiler/rustc_mir/src/const_eval/eval_queries.rs @@ -1,8 +1,8 @@ use super::{CompileTimeEvalContext, CompileTimeInterpreter, ConstEvalErr, MemoryExtra}; use crate::interpret::eval_nullary_intrinsic; use crate::interpret::{ - intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, GlobalId, Immediate, - InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, Scalar, + intern_const_alloc_recursive, Allocation, ConstAlloc, ConstValue, CtfeValidationMode, GlobalId, + Immediate, InternKind, InterpCx, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, Scalar, ScalarMaybeUninit, StackPopCleanup, }; @@ -59,23 +59,15 @@ fn eval_body_using_ecx<'mir, 'tcx>( ecx.run()?; // Intern the result - // FIXME: since the DefId of a promoted is the DefId of its owner, this - // means that promoteds in statics are actually interned like statics! - // However, this is also currently crucial because we promote mutable - // non-empty slices in statics to extend their lifetime, and this - // ensures that they are put into a mutable allocation. - // For other kinds of promoteds in statics (like array initializers), this is rather silly. - let intern_kind = match tcx.static_mutability(cid.instance.def_id()) { - Some(m) => InternKind::Static(m), - None if cid.promoted.is_some() => InternKind::Promoted, - _ => InternKind::Constant, + let intern_kind = if cid.promoted.is_some() { + InternKind::Promoted + } else { + match tcx.static_mutability(cid.instance.def_id()) { + Some(m) => InternKind::Static(m), + None => InternKind::Constant, + } }; - intern_const_alloc_recursive( - ecx, - intern_kind, - ret, - body.ignore_interior_mut_in_const_validation, - ); + intern_const_alloc_recursive(ecx, intern_kind, ret); debug!("eval_body_using_ecx done: {:?}", *ret); Ok(ret) @@ -376,16 +368,23 @@ pub fn eval_to_allocation_raw_provider<'tcx>( // Since evaluation had no errors, valiate the resulting constant: let validation = try { // FIXME do not validate promoteds until a decision on - // https://github.com/rust-lang/rust/issues/67465 is made + // https://github.com/rust-lang/rust/issues/67465 and + // https://github.com/rust-lang/rust/issues/67534 is made. + // Promoteds can contain unexpected `UnsafeCell` and reference `static`s, but their + // otherwise restricted form ensures that this is still sound. We just lose the + // extra safety net of some of the dynamic checks. They can also contain invalid + // values, but since we do not usually check intermediate results of a computation + // for validity, it might be surprising to do that here. if cid.promoted.is_none() { let mut ref_tracking = RefTracking::new(mplace); + let mut inner = false; while let Some((mplace, path)) = ref_tracking.todo.pop() { - ecx.const_validate_operand( - mplace.into(), - path, - &mut ref_tracking, - /*may_ref_to_static*/ ecx.memory.extra.can_access_statics, - )?; + let mode = match tcx.static_mutability(cid.instance.def_id()) { + Some(_) => CtfeValidationMode::Regular, // a `static` + None => CtfeValidationMode::Const { inner }, + }; + ecx.const_validate_operand(mplace.into(), path, &mut ref_tracking, mode)?; + inner = true; } } }; diff --git a/compiler/rustc_mir/src/const_eval/machine.rs b/compiler/rustc_mir/src/const_eval/machine.rs index 73ca7e0d471..7e2cae09481 100644 --- a/compiler/rustc_mir/src/const_eval/machine.rs +++ b/compiler/rustc_mir/src/const_eval/machine.rs @@ -70,9 +70,10 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> { ) -> InterpResult<'tcx> { let def_id = instance.def_id(); if Some(def_id) == self.tcx.lang_items().panic_fn() + || Some(def_id) == self.tcx.lang_items().panic_str() || Some(def_id) == self.tcx.lang_items().begin_panic_fn() { - // &'static str + // &str assert!(args.len() == 1); let msg_place = self.deref_operand(args[0])?; diff --git a/compiler/rustc_mir/src/const_eval/mod.rs b/compiler/rustc_mir/src/const_eval/mod.rs index 4b235e1aa4a..11a211ef7b3 100644 --- a/compiler/rustc_mir/src/const_eval/mod.rs +++ b/compiler/rustc_mir/src/const_eval/mod.rs @@ -29,7 +29,7 @@ pub(crate) fn const_caller_location( let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), false); let loc_place = ecx.alloc_caller_location(file, line, col); - intern_const_alloc_recursive(&mut ecx, InternKind::Constant, loc_place, false); + intern_const_alloc_recursive(&mut ecx, InternKind::Constant, loc_place); ConstValue::Scalar(loc_place.ptr) } diff --git a/compiler/rustc_mir/src/dataflow/impls/borrows.rs b/compiler/rustc_mir/src/dataflow/impls/borrows.rs index 0be13b6ba81..6b7889c4d9e 100644 --- a/compiler/rustc_mir/src/dataflow/impls/borrows.rs +++ b/compiler/rustc_mir/src/dataflow/impls/borrows.rs @@ -177,7 +177,7 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> { // // We are careful always to call this function *before* we // set up the gen-bits for the statement or - // termanator. That way, if the effect of the statement or + // terminator. That way, if the effect of the statement or // terminator *does* introduce a new loan of the same // region, then setting that gen-bit will override any // potential kill introduced here. diff --git a/compiler/rustc_mir/src/dataflow/impls/liveness.rs b/compiler/rustc_mir/src/dataflow/impls/liveness.rs index b0da28156d1..a2b0713cd7d 100644 --- a/compiler/rustc_mir/src/dataflow/impls/liveness.rs +++ b/compiler/rustc_mir/src/dataflow/impls/liveness.rs @@ -8,7 +8,7 @@ use crate::dataflow::{AnalysisDomain, Backward, GenKill, GenKillAnalysis}; /// /// This analysis considers references as being used only at the point of the /// borrow. In other words, this analysis does not track uses because of references that already -/// exist. See [this `mir-datalow` test][flow-test] for an example. You almost never want to use +/// exist. See [this `mir-dataflow` test][flow-test] for an example. You almost never want to use /// this analysis without also looking at the results of [`MaybeBorrowedLocals`]. /// /// [`MaybeBorrowedLocals`]: ../struct.MaybeBorrowedLocals.html @@ -134,7 +134,7 @@ impl DefUse { // `MutatingUseContext::Call` and `MutatingUseContext::Yield` indicate that this is the // destination place for a `Call` return or `Yield` resume respectively. Since this is - // only a `Def` when the function returns succesfully, we handle this case separately + // only a `Def` when the function returns successfully, we handle this case separately // in `call_return_effect` above. PlaceContext::MutatingUse(MutatingUseContext::Call | MutatingUseContext::Yield) => None, diff --git a/compiler/rustc_mir/src/interpret/intern.rs b/compiler/rustc_mir/src/interpret/intern.rs index dd5e9c99774..5e5c74a3723 100644 --- a/compiler/rustc_mir/src/interpret/intern.rs +++ b/compiler/rustc_mir/src/interpret/intern.rs @@ -2,12 +2,23 @@ //! //! After a const evaluation has computed a value, before we destroy the const evaluator's session //! memory, we need to extract all memory allocations to the global memory pool so they stay around. +//! +//! In principle, this is not very complicated: we recursively walk the final value, follow all the +//! pointers, and move all reachable allocations to the global `tcx` memory. The only complication +//! is picking the right mutability for the allocations in a `static` initializer: we want to make +//! as many allocations as possible immutable so LLVM can put them into read-only memory. At the +//! same time, we need to make memory that could be mutated by the program mutable to avoid +//! incorrect compilations. To achieve this, we do a type-based traversal of the final value, +//! tracking mutable and shared references and `UnsafeCell` to determine the current mutability. +//! (In principle, we could skip this type-based part for `const` and promoteds, as they need to be +//! always immutable. At least for `const` however we use this opportunity to reject any `const` +//! that contains allocations whose mutability we cannot identify.) use super::validity::RefTracking; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir as hir; use rustc_middle::mir::interpret::InterpResult; -use rustc_middle::ty::{self, layout::TyAndLayout, query::TyCtxtAt, Ty}; +use rustc_middle::ty::{self, layout::TyAndLayout, Ty}; use rustc_target::abi::Size; use rustc_ast::Mutability; @@ -33,17 +44,13 @@ struct InternVisitor<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> { /// A list of all encountered allocations. After type-based interning, we traverse this list to /// also intern allocations that are only referenced by a raw pointer or inside a union. leftover_allocations: &'rt mut FxHashSet<AllocId>, - /// The root kind of the value that we're looking at. This field is never mutated and only used - /// for sanity assertions that will ICE when `const_qualif` screws up. + /// The root kind of the value that we're looking at. This field is never mutated for a + /// particular allocation. It is primarily used to make as many allocations as possible + /// read-only so LLVM can place them in const memory. mode: InternMode, /// This field stores whether we are *currently* inside an `UnsafeCell`. This can affect /// the intern mode of references we encounter. inside_unsafe_cell: bool, - - /// This flag is to avoid triggering UnsafeCells are not allowed behind references in constants - /// for promoteds. - /// It's a copy of `mir::Body`'s ignore_interior_mut_in_const_validation field - ignore_interior_mut_in_const: bool, } #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)] @@ -52,22 +59,14 @@ enum InternMode { /// this is *immutable*, and below mutable references inside an `UnsafeCell`, this /// is *mutable*. Static(hir::Mutability), - /// The "base value" of a const, which can have `UnsafeCell` (as in `const FOO: Cell<i32>`), - /// but that interior mutability is simply ignored. - ConstBase, - /// The "inner values" of a const with references, where `UnsafeCell` is an error. - ConstInner, + /// A `const`. + Const, } /// Signalling data structure to ensure we don't recurse /// into the memory of other constants or statics struct IsStaticOrFn; -fn mutable_memory_in_const(tcx: TyCtxtAt<'_>, kind: &str) { - // FIXME: show this in validation instead so we can point at where in the value the error is? - tcx.sess.span_err(tcx.span, &format!("mutable memory ({}) is not allowed in constant", kind)); -} - /// Intern an allocation without looking at its children. /// `mode` is the mode of the environment where we found this pointer. /// `mutablity` is the mutability of the place to be interned; even if that says @@ -113,8 +112,8 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>>( // For this, we need to take into account `UnsafeCell`. When `ty` is `None`, we assume // no interior mutability. let frozen = ty.map_or(true, |ty| ty.is_freeze(ecx.tcx, ecx.param_env)); - // For statics, allocation mutability is the combination of the place mutability and - // the type mutability. + // For statics, allocation mutability is the combination of place mutability and + // type mutability. // The entire allocation needs to be mutable if it contains an `UnsafeCell` anywhere. let immutable = mutability == Mutability::Not && frozen; if immutable { @@ -128,9 +127,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>>( // See const_eval::machine::MemoryExtra::can_access_statics for why // immutability is so important. - // There are no sensible checks we can do here; grep for `mutable_memory_in_const` to - // find the checks we are doing elsewhere to avoid even getting here for memory - // that "wants" to be mutable. + // Validation will ensure that there is no `UnsafeCell` on an immutable allocation. alloc.mutability = Mutability::Not; }; // link the alloc id to the actual allocation @@ -166,17 +163,13 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir mplace: MPlaceTy<'tcx>, fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>, ) -> InterpResult<'tcx> { + // ZSTs cannot contain pointers, so we can skip them. + if mplace.layout.is_zst() { + return Ok(()); + } + if let Some(def) = mplace.layout.ty.ty_adt_def() { if Some(def.did) == self.ecx.tcx.lang_items().unsafe_cell_type() { - if self.mode == InternMode::ConstInner && !self.ignore_interior_mut_in_const { - // We do not actually make this memory mutable. But in case the user - // *expected* it to be mutable, make sure we error. This is just a - // sanity check to prevent users from accidentally exploiting the UB - // they caused. It also helps us to find cases where const-checking - // failed to prevent an `UnsafeCell` (but as `ignore_interior_mut_in_const` - // shows that part is not airtight). - mutable_memory_in_const(self.ecx.tcx, "`UnsafeCell`"); - } // We are crossing over an `UnsafeCell`, we can mutate again. This means that // References we encounter inside here are interned as pointing to mutable // allocations. @@ -187,6 +180,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir return walked; } } + self.walk_aggregate(mplace, fields) } @@ -203,13 +197,12 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir if let ty::Dynamic(..) = tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind() { - // Validation will error (with a better message) on an invalid vtable pointer - // so we can safely not do anything if this is not a real pointer. if let Scalar::Ptr(vtable) = mplace.meta.unwrap_meta() { // Explicitly choose const mode here, since vtables are immutable, even // if the reference of the fat pointer is mutable. - self.intern_shallow(vtable.alloc_id, InternMode::ConstInner, None); + self.intern_shallow(vtable.alloc_id, InternMode::Const, None); } else { + // Validation will error (with a better message) on an invalid vtable pointer. // Let validation show the error message, but make sure it *does* error. tcx.sess .delay_span_bug(tcx.span, "vtables pointers cannot be integer pointers"); @@ -218,7 +211,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir // Check if we have encountered this pointer+layout combination before. // Only recurse for allocation-backed pointers. if let Scalar::Ptr(ptr) = mplace.ptr { - // Compute the mode with which we intern this. + // Compute the mode with which we intern this. Our goal here is to make as many + // statics as we can immutable so they can be placed in read-only memory by LLVM. let ref_mode = match self.mode { InternMode::Static(mutbl) => { // In statics, merge outer mutability with reference mutability and @@ -237,8 +231,13 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir } Mutability::Not => { // A shared reference, things become immutable. - // We do *not* consier `freeze` here -- that is done more precisely - // when traversing the referenced data (by tracking `UnsafeCell`). + // We do *not* consider `freeze` here: `intern_shallow` considers + // `freeze` for the actual mutability of this allocation; the intern + // mode for references contained in this allocation is tracked more + // precisely when traversing the referenced data (by tracking + // `UnsafeCell`). This makes sure that `&(&i32, &Cell<i32>)` still + // has the left inner reference interned into a read-only + // allocation. InternMode::Static(Mutability::Not) } Mutability::Mut => { @@ -247,27 +246,11 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir } } } - InternMode::ConstBase | InternMode::ConstInner => { - // Ignore `UnsafeCell`, everything is immutable. Do some sanity checking - // for mutable references that we encounter -- they must all be ZST. - // This helps to prevent users from accidentally exploiting UB that they - // caused (by somehow getting a mutable reference in a `const`). - if ref_mutability == Mutability::Mut { - match referenced_ty.kind() { - ty::Array(_, n) if n.eval_usize(*tcx, self.ecx.param_env) == 0 => {} - ty::Slice(_) - if mplace.meta.unwrap_meta().to_machine_usize(self.ecx)? - == 0 => {} - _ => mutable_memory_in_const(tcx, "`&mut`"), - } - } else { - // A shared reference. We cannot check `freeze` here due to references - // like `&dyn Trait` that are actually immutable. We do check for - // concrete `UnsafeCell` when traversing the pointee though (if it is - // a new allocation, not yet interned). - } - // Go on with the "inner" rules. - InternMode::ConstInner + InternMode::Const => { + // Ignore `UnsafeCell`, everything is immutable. Validity does some sanity + // checking for mutable references that we encounter -- they must all be + // ZST. + InternMode::Const } }; match self.intern_shallow(ptr.alloc_id, ref_mode, Some(referenced_ty)) { @@ -306,7 +289,6 @@ pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>( ecx: &mut InterpCx<'mir, 'tcx, M>, intern_kind: InternKind, ret: MPlaceTy<'tcx>, - ignore_interior_mut_in_const: bool, ) where 'tcx: 'mir, { @@ -315,7 +297,7 @@ pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>( InternKind::Static(mutbl) => InternMode::Static(mutbl), // `Constant` includes array lengths. // `Promoted` includes non-`Copy` array initializers and `rustc_args_required_const` arguments. - InternKind::Constant | InternKind::Promoted => InternMode::ConstBase, + InternKind::Constant | InternKind::Promoted => InternMode::Const, }; // Type based interning. @@ -345,7 +327,6 @@ pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>( ecx, mode, leftover_allocations, - ignore_interior_mut_in_const, inside_unsafe_cell: false, } .visit_value(mplace); diff --git a/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs index d9be28cf9db..5c917f00d15 100644 --- a/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs +++ b/compiler/rustc_mir/src/interpret/intrinsics/caller_location.rs @@ -15,38 +15,61 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a /// frame which is not `#[track_caller]`. crate fn find_closest_untracked_caller_location(&self) -> Span { - let frame = self - .stack() - .iter() - .rev() - // Find first non-`#[track_caller]` frame. - .find(|frame| { + for frame in self.stack().iter().rev() { + debug!("find_closest_untracked_caller_location: checking frame {:?}", frame.instance); + + // Assert that the frame we look at is actually executing code currently + // (`loc` is `Err` when we are unwinding and the frame does not require cleanup). + let loc = frame.loc.unwrap(); + + // This could be a non-`Call` terminator (such as `Drop`), or not a terminator at all + // (such as `box`). Use the normal span by default. + let mut source_info = *frame.body.source_info(loc); + + // If this is a `Call` terminator, use the `fn_span` instead. + let block = &frame.body.basic_blocks()[loc.block]; + if loc.statement_index == block.statements.len() { debug!( - "find_closest_untracked_caller_location: checking frame {:?}", - frame.instance + "find_closest_untracked_caller_location: got terminator {:?} ({:?})", + block.terminator(), + block.terminator().kind ); - !frame.instance.def.requires_caller_location(*self.tcx) - }) - // Assert that there is always such a frame. - .unwrap(); - // Assert that the frame we look at is actually executing code currently - // (`loc` is `Err` when we are unwinding and the frame does not require cleanup). - let loc = frame.loc.unwrap(); - // If this is a `Call` terminator, use the `fn_span` instead. - let block = &frame.body.basic_blocks()[loc.block]; - if loc.statement_index == block.statements.len() { - debug!( - "find_closest_untracked_caller_location:: got terminator {:?} ({:?})", - block.terminator(), - block.terminator().kind - ); - if let TerminatorKind::Call { fn_span, .. } = block.terminator().kind { - return fn_span; + if let TerminatorKind::Call { fn_span, .. } = block.terminator().kind { + source_info.span = fn_span; + } + } + + // Walk up the `SourceScope`s, in case some of them are from MIR inlining. + // If so, the starting `source_info.span` is in the innermost inlined + // function, and will be replaced with outer callsite spans as long + // as the inlined functions were `#[track_caller]`. + loop { + let scope_data = &frame.body.source_scopes[source_info.scope]; + + if let Some((callee, callsite_span)) = scope_data.inlined { + // Stop inside the most nested non-`#[track_caller]` function, + // before ever reaching its caller (which is irrelevant). + if !callee.def.requires_caller_location(*self.tcx) { + return source_info.span; + } + source_info.span = callsite_span; + } + + // Skip past all of the parents with `inlined: None`. + match scope_data.inlined_parent_scope { + Some(parent) => source_info.scope = parent, + None => break, + } + } + + // Stop inside the most nested non-`#[track_caller]` function, + // before ever reaching its caller (which is irrelevant). + if !frame.instance.def.requires_caller_location(*self.tcx) { + return source_info.span; } } - // This is a different terminator (such as `Drop`) or not a terminator at all - // (such as `box`). Use the normal span. - frame.body.source_info(loc).span + + bug!("no non-`#[track_caller]` frame found") } /// Allocate a `const core::panic::Location` with the provided filename and line/column numbers. diff --git a/compiler/rustc_mir/src/interpret/mod.rs b/compiler/rustc_mir/src/interpret/mod.rs index a931b0bbe97..a29ef117ace 100644 --- a/compiler/rustc_mir/src/interpret/mod.rs +++ b/compiler/rustc_mir/src/interpret/mod.rs @@ -24,7 +24,7 @@ pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackP pub use self::memory::{AllocCheck, FnVal, Memory, MemoryKind}; pub use self::operand::{ImmTy, Immediate, OpTy, Operand}; pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy}; -pub use self::validity::RefTracking; +pub use self::validity::{CtfeValidationMode, RefTracking}; pub use self::visitor::{MutValueVisitor, ValueVisitor}; crate use self::intrinsics::eval_nullary_intrinsic; diff --git a/compiler/rustc_mir/src/interpret/operand.rs b/compiler/rustc_mir/src/interpret/operand.rs index d8f27ec9545..3c68b1c8355 100644 --- a/compiler/rustc_mir/src/interpret/operand.rs +++ b/compiler/rustc_mir/src/interpret/operand.rs @@ -117,7 +117,7 @@ impl<Tag: Copy> std::fmt::Display for ImmTy<'tcx, Tag> { ty::tls::with(|tcx| { match self.imm { Immediate::Scalar(s) => { - if let Some(ty) = tcx.lift(&self.layout.ty) { + if let Some(ty) = tcx.lift(self.layout.ty) { let cx = FmtPrinter::new(tcx, f, Namespace::ValueNS); p(cx, s, ty)?; return Ok(()); diff --git a/compiler/rustc_mir/src/interpret/util.rs b/compiler/rustc_mir/src/interpret/util.rs index fc5a25ffbf2..fce5553c993 100644 --- a/compiler/rustc_mir/src/interpret/util.rs +++ b/compiler/rustc_mir/src/interpret/util.rs @@ -1,6 +1,7 @@ use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitor}; use std::convert::TryInto; +use std::ops::ControlFlow; /// Returns `true` if a used generic parameter requires substitution. crate fn ensure_monomorphic_enough<'tcx, T>(tcx: TyCtxt<'tcx>, ty: T) -> InterpResult<'tcx> @@ -17,24 +18,24 @@ where }; impl<'tcx> TypeVisitor<'tcx> for UsedParamsNeedSubstVisitor<'tcx> { - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { if !c.needs_subst() { - return false; + return ControlFlow::CONTINUE; } match c.val { - ty::ConstKind::Param(..) => true, + ty::ConstKind::Param(..) => ControlFlow::BREAK, _ => c.super_visit_with(self), } } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { if !ty.needs_subst() { - return false; + return ControlFlow::CONTINUE; } match *ty.kind() { - ty::Param(_) => true, + ty::Param(_) => ControlFlow::BREAK, ty::Closure(def_id, substs) | ty::Generator(def_id, substs, ..) | ty::FnDef(def_id, substs) => { @@ -50,11 +51,7 @@ where match (is_used, subst.needs_subst()) { // Just in case there are closures or generators within this subst, // recurse. - (true, true) if subst.super_visit_with(self) => { - // Only return when we find a parameter so the remaining substs - // are not skipped. - return true; - } + (true, true) => return subst.super_visit_with(self), // Confirm that polymorphization replaced the parameter with // `ty::Param`/`ty::ConstKind::Param`. (false, true) if cfg!(debug_assertions) => match subst.unpack() { @@ -69,7 +66,7 @@ where _ => {} } } - false + ControlFlow::CONTINUE } _ => ty.super_visit_with(self), } @@ -77,7 +74,7 @@ where } let mut vis = UsedParamsNeedSubstVisitor { tcx }; - if ty.visit_with(&mut vis) { + if ty.visit_with(&mut vis).is_break() { throw_inval!(TooGeneric); } else { Ok(()) diff --git a/compiler/rustc_mir/src/interpret/validity.rs b/compiler/rustc_mir/src/interpret/validity.rs index 2b83e1c8134..2d235d65c4d 100644 --- a/compiler/rustc_mir/src/interpret/validity.rs +++ b/compiler/rustc_mir/src/interpret/validity.rs @@ -113,6 +113,17 @@ pub enum PathElem { DynDowncast, } +/// Extra things to check for during validation of CTFE results. +pub enum CtfeValidationMode { + /// Regular validation, nothing special happening. + Regular, + /// Validation of a `const`. `inner` says if this is an inner, indirect allocation (as opposed + /// to the top-level const allocation). + /// Being an inner allocation makes a difference because the top-level allocation of a `const` + /// is copied for each use, but the inner allocations are implicitly shared. + Const { inner: bool }, +} + /// State for tracking recursive validation of references pub struct RefTracking<T, PATH = ()> { pub seen: FxHashSet<T>, @@ -202,9 +213,9 @@ struct ValidityVisitor<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// starts must not be changed! `visit_fields` and `visit_array` rely on /// this stack discipline. path: Vec<PathElem>, - ref_tracking_for_consts: - Option<&'rt mut RefTracking<MPlaceTy<'tcx, M::PointerTag>, Vec<PathElem>>>, - may_ref_to_static: bool, + ref_tracking: Option<&'rt mut RefTracking<MPlaceTy<'tcx, M::PointerTag>, Vec<PathElem>>>, + /// `None` indicates this is not validating for CTFE (but for runtime). + ctfe_mode: Option<CtfeValidationMode>, ecx: &'rt InterpCx<'mir, 'tcx, M>, } @@ -418,7 +429,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' { "a dangling {} (use-after-free)", kind }, ); // Recursive checking - if let Some(ref mut ref_tracking) = self.ref_tracking_for_consts { + if let Some(ref mut ref_tracking) = self.ref_tracking { if let Some(ptr) = ptr { // not a ZST // Skip validation entirely for some external statics @@ -426,19 +437,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' if let Some(GlobalAlloc::Static(did)) = alloc_kind { assert!(!self.ecx.tcx.is_thread_local_static(did)); assert!(self.ecx.tcx.is_static(did)); - if self.may_ref_to_static { - // We skip checking other statics. These statics must be sound by - // themselves, and the only way to get broken statics here is by using - // unsafe code. - // The reasons we don't check other statics is twofold. For one, in all - // sound cases, the static was already validated on its own, and second, we - // trigger cycle errors if we try to compute the value of the other static - // and that static refers back to us. - // We might miss const-invalid data, - // but things are still sound otherwise (in particular re: consts - // referring to statics). - return Ok(()); - } else { + if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. })) { // See const_eval::machine::MemoryExtra::can_access_statics for why // this check is so important. // This check is reachable when the const just referenced the static, @@ -447,6 +446,17 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' { "a {} pointing to a static variable", kind } ); } + // We skip checking other statics. These statics must be sound by + // themselves, and the only way to get broken statics here is by using + // unsafe code. + // The reasons we don't check other statics is twofold. For one, in all + // sound cases, the static was already validated on its own, and second, we + // trigger cycle errors if we try to compute the value of the other static + // and that static refers back to us. + // We might miss const-invalid data, + // but things are still sound otherwise (in particular re: consts + // referring to statics). + return Ok(()); } } // Proceed recursively even for ZST, no reason to skip them! @@ -504,7 +514,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let value = self.ecx.read_scalar(value)?; // NOTE: Keep this in sync with the array optimization for int/float // types below! - if self.ref_tracking_for_consts.is_some() { + if self.ctfe_mode.is_some() { // Integers/floats in CTFE: Must be scalar bits, pointers are dangerous let is_bits = value.check_init().map_or(false, |v| v.is_bits()); if !is_bits { @@ -532,7 +542,17 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' } Ok(true) } - ty::Ref(..) => { + ty::Ref(_, ty, mutbl) => { + if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { .. })) + && *mutbl == hir::Mutability::Mut + { + // A mutable reference inside a const? That does not seem right (except if it is + // a ZST). + let layout = self.ecx.layout_of(ty)?; + if !layout.is_zst() { + throw_validation_failure!(self.path, { "mutable reference in a `const`" }); + } + } self.check_safe_pointer(value, "reference")?; Ok(true) } @@ -559,9 +579,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' // Nothing to check. Ok(true) } - // The above should be all the (inhabited) primitive types. The rest is compound, we + // The above should be all the primitive types. The rest is compound, we // check them by visiting their fields/variants. - // (`Str` UTF-8 check happens in `visit_aggregate`, too.) ty::Adt(..) | ty::Tuple(..) | ty::Array(..) @@ -723,6 +742,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // Sanity check: `builtin_deref` does not know any pointers that are not primitive. assert!(op.layout.ty.builtin_deref(true).is_none()); + // Special check preventing `UnsafeCell` in constants + if let Some(def) = op.layout.ty.ty_adt_def() { + if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { inner: true })) + && Some(def.did) == self.ecx.tcx.lang_items().unsafe_cell_type() + { + throw_validation_failure!(self.path, { "`UnsafeCell` in a `const`" }); + } + } + // Recursively walk the value at its type. self.walk_value(op)?; @@ -775,17 +803,13 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> ); } ty::Array(tys, ..) | ty::Slice(tys) - if { - // This optimization applies for types that can hold arbitrary bytes (such as - // integer and floating point types) or for structs or tuples with no fields. - // FIXME(wesleywiser) This logic could be extended further to arbitrary structs - // or tuples made up of integer/floating point types or inhabited ZSTs with no - // padding. - match tys.kind() { - ty::Int(..) | ty::Uint(..) | ty::Float(..) => true, - _ => false, - } - } => + // This optimization applies for types that can hold arbitrary bytes (such as + // integer and floating point types) or for structs or tuples with no fields. + // FIXME(wesleywiser) This logic could be extended further to arbitrary structs + // or tuples made up of integer/floating point types or inhabited ZSTs with no + // padding. + if matches!(tys.kind(), ty::Int(..) | ty::Uint(..) | ty::Float(..)) + => { // Optimized handling for arrays of integer/float type. @@ -818,7 +842,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> self.ecx, ptr, size, - /*allow_uninit_and_ptr*/ self.ref_tracking_for_consts.is_none(), + /*allow_uninit_and_ptr*/ self.ctfe_mode.is_none(), ) { // In the happy case, we needn't check anything else. Ok(()) => {} @@ -853,7 +877,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // of an array and not all of them, because there's only a single value of a specific // ZST type, so either validation fails for all elements or none. ty::Array(tys, ..) | ty::Slice(tys) if self.ecx.layout_of(tys)?.is_zst() => { - // Validate just the first element + // Validate just the first element (if any). self.walk_aggregate(op, fields.take(1))? } _ => { @@ -869,16 +893,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &self, op: OpTy<'tcx, M::PointerTag>, path: Vec<PathElem>, - ref_tracking_for_consts: Option< - &mut RefTracking<MPlaceTy<'tcx, M::PointerTag>, Vec<PathElem>>, - >, - may_ref_to_static: bool, + ref_tracking: Option<&mut RefTracking<MPlaceTy<'tcx, M::PointerTag>, Vec<PathElem>>>, + ctfe_mode: Option<CtfeValidationMode>, ) -> InterpResult<'tcx> { trace!("validate_operand_internal: {:?}, {:?}", *op, op.layout.ty); // Construct a visitor - let mut visitor = - ValidityVisitor { path, ref_tracking_for_consts, may_ref_to_static, ecx: self }; + let mut visitor = ValidityVisitor { path, ref_tracking, ctfe_mode, ecx: self }; // Try to cast to ptr *once* instead of all the time. let op = self.force_op_ptr(op).unwrap_or(op); @@ -906,16 +927,18 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// `ref_tracking` is used to record references that we encounter so that they /// can be checked recursively by an outside driving loop. /// - /// `may_ref_to_static` controls whether references are allowed to point to statics. + /// `constant` controls whether this must satisfy the rules for constants: + /// - no pointers to statics. + /// - no `UnsafeCell` or non-ZST `&mut`. #[inline(always)] pub fn const_validate_operand( &self, op: OpTy<'tcx, M::PointerTag>, path: Vec<PathElem>, ref_tracking: &mut RefTracking<MPlaceTy<'tcx, M::PointerTag>, Vec<PathElem>>, - may_ref_to_static: bool, + ctfe_mode: CtfeValidationMode, ) -> InterpResult<'tcx> { - self.validate_operand_internal(op, path, Some(ref_tracking), may_ref_to_static) + self.validate_operand_internal(op, path, Some(ref_tracking), Some(ctfe_mode)) } /// This function checks the data at `op` to be runtime-valid. @@ -923,6 +946,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// It will error if the bits at the destination do not match the ones described by the layout. #[inline(always)] pub fn validate_operand(&self, op: OpTy<'tcx, M::PointerTag>) -> InterpResult<'tcx> { - self.validate_operand_internal(op, vec![], None, false) + self.validate_operand_internal(op, vec![], None, None) } } diff --git a/compiler/rustc_mir/src/lib.rs b/compiler/rustc_mir/src/lib.rs index c00c6860903..2ed115b1297 100644 --- a/compiler/rustc_mir/src/lib.rs +++ b/compiler/rustc_mir/src/lib.rs @@ -27,6 +27,7 @@ Rust MIR: a lowered representation of Rust. #![feature(option_expect_none)] #![feature(or_patterns)] #![feature(once_cell)] +#![feature(control_flow_enum)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_mir/src/monomorphize/polymorphize.rs b/compiler/rustc_mir/src/monomorphize/polymorphize.rs index 3f6f117acdc..c2ebc954a22 100644 --- a/compiler/rustc_mir/src/monomorphize/polymorphize.rs +++ b/compiler/rustc_mir/src/monomorphize/polymorphize.rs @@ -20,6 +20,7 @@ use rustc_middle::ty::{ }; use rustc_span::symbol::sym; use std::convert::TryInto; +use std::ops::ControlFlow; /// Provide implementations of queries relating to polymorphization analysis. pub fn provide(providers: &mut Providers) { @@ -138,7 +139,7 @@ fn mark_used_by_predicates<'tcx>( // predicate is used. let any_param_used = { let mut vis = HasUsedGenericParams { unused_parameters }; - predicate.visit_with(&mut vis) + predicate.visit_with(&mut vis).is_break() }; if any_param_used { @@ -249,17 +250,17 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> { } impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> { - fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> ControlFlow<()> { debug!("visit_const: c={:?}", c); if !c.has_param_types_or_consts() { - return false; + return ControlFlow::CONTINUE; } match c.val { ty::ConstKind::Param(param) => { debug!("visit_const: param={:?}", param); self.unused_parameters.clear(param.index); - false + ControlFlow::CONTINUE } ty::ConstKind::Unevaluated(def, _, Some(p)) // Avoid considering `T` unused when constants are of the form: @@ -270,22 +271,22 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> { // the generic parameters, instead, traverse the promoted MIR. let promoted = self.tcx.promoted_mir(def.did); self.visit_body(&promoted[p]); - false + ControlFlow::CONTINUE } ty::ConstKind::Unevaluated(def, unevaluated_substs, None) if self.tcx.def_kind(def.did) == DefKind::AnonConst => { self.visit_child_body(def.did, unevaluated_substs); - false + ControlFlow::CONTINUE } _ => c.super_visit_with(self), } } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { debug!("visit_ty: ty={:?}", ty); if !ty.has_param_types_or_consts() { - return false; + return ControlFlow::CONTINUE; } match *ty.kind() { @@ -293,18 +294,18 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> { debug!("visit_ty: def_id={:?}", def_id); // Avoid cycle errors with generators. if def_id == self.def_id { - return false; + return ControlFlow::CONTINUE; } // Consider any generic parameters used by any closures/generators as used in the // parent. self.visit_child_body(def_id, substs); - false + ControlFlow::CONTINUE } ty::Param(param) => { debug!("visit_ty: param={:?}", param); self.unused_parameters.clear(param.index); - false + ControlFlow::CONTINUE } _ => ty.super_visit_with(self), } @@ -317,28 +318,38 @@ struct HasUsedGenericParams<'a> { } impl<'a, 'tcx> TypeVisitor<'tcx> for HasUsedGenericParams<'a> { - fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx Const<'tcx>) -> ControlFlow<()> { debug!("visit_const: c={:?}", c); if !c.has_param_types_or_consts() { - return false; + return ControlFlow::CONTINUE; } match c.val { ty::ConstKind::Param(param) => { - !self.unused_parameters.contains(param.index).unwrap_or(false) + if self.unused_parameters.contains(param.index).unwrap_or(false) { + ControlFlow::CONTINUE + } else { + ControlFlow::BREAK + } } _ => c.super_visit_with(self), } } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { debug!("visit_ty: ty={:?}", ty); if !ty.has_param_types_or_consts() { - return false; + return ControlFlow::CONTINUE; } match ty.kind() { - ty::Param(param) => !self.unused_parameters.contains(param.index).unwrap_or(false), + ty::Param(param) => { + if self.unused_parameters.contains(param.index).unwrap_or(false) { + ControlFlow::CONTINUE + } else { + ControlFlow::BREAK + } + } _ => ty.super_visit_with(self), } } diff --git a/compiler/rustc_mir/src/shim.rs b/compiler/rustc_mir/src/shim.rs index 5431d22e703..b2fa4b11f36 100644 --- a/compiler/rustc_mir/src/shim.rs +++ b/compiler/rustc_mir/src/shim.rs @@ -212,7 +212,13 @@ fn new_body<'tcx>( source, basic_blocks, IndexVec::from_elem_n( - SourceScopeData { span, parent_scope: None, local_data: ClearCrossCrate::Clear }, + SourceScopeData { + span, + parent_scope: None, + inlined: None, + inlined_parent_scope: None, + local_data: ClearCrossCrate::Clear, + }, 1, ), local_decls, diff --git a/compiler/rustc_mir/src/transform/check_consts/mod.rs b/compiler/rustc_mir/src/transform/check_consts/mod.rs index 33815ceba62..ba7bea4ac54 100644 --- a/compiler/rustc_mir/src/transform/check_consts/mod.rs +++ b/compiler/rustc_mir/src/transform/check_consts/mod.rs @@ -74,12 +74,18 @@ impl ConstCx<'mir, 'tcx> { /// Returns `true` if this `DefId` points to one of the official `panic` lang items. pub fn is_lang_panic_fn(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool { - Some(def_id) == tcx.lang_items().panic_fn() || Some(def_id) == tcx.lang_items().begin_panic_fn() + Some(def_id) == tcx.lang_items().panic_fn() + || Some(def_id) == tcx.lang_items().panic_str() + || Some(def_id) == tcx.lang_items().begin_panic_fn() } -pub fn allow_internal_unstable(tcx: TyCtxt<'tcx>, def_id: DefId, feature_gate: Symbol) -> bool { +pub fn rustc_allow_const_fn_unstable( + tcx: TyCtxt<'tcx>, + def_id: DefId, + feature_gate: Symbol, +) -> bool { let attrs = tcx.get_attrs(def_id); - attr::allow_internal_unstable(&tcx.sess, attrs) + attr::rustc_allow_const_fn_unstable(&tcx.sess, attrs) .map_or(false, |mut features| features.any(|name| name == feature_gate)) } diff --git a/compiler/rustc_mir/src/transform/check_consts/ops.rs b/compiler/rustc_mir/src/transform/check_consts/ops.rs index 63b20c7c027..bd51136b8db 100644 --- a/compiler/rustc_mir/src/transform/check_consts/ops.rs +++ b/compiler/rustc_mir/src/transform/check_consts/ops.rs @@ -224,7 +224,8 @@ impl NonConstOp for CellBorrow { } #[derive(Debug)] -pub struct MutBorrow; +pub struct MutBorrow(pub hir::BorrowKind); + impl NonConstOp for MutBorrow { fn status_in_item(&self, ccx: &ConstCx<'_, '_>) -> Status { // Forbid everywhere except in const fn with a feature gate @@ -236,22 +237,28 @@ impl NonConstOp for MutBorrow { } fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { + let raw = match self.0 { + hir::BorrowKind::Raw => "raw ", + hir::BorrowKind::Ref => "", + }; + let mut err = if ccx.const_kind() == hir::ConstContext::ConstFn { feature_err( &ccx.tcx.sess.parse_sess, sym::const_mut_refs, span, - &format!("mutable references are not allowed in {}s", ccx.const_kind()), + &format!("{}mutable references are not allowed in {}s", raw, ccx.const_kind()), ) } else { let mut err = struct_span_err!( ccx.tcx.sess, span, E0764, - "mutable references are not allowed in {}s", + "{}mutable references are not allowed in {}s", + raw, ccx.const_kind(), ); - err.span_label(span, format!("`&mut` is only allowed in `const fn`")); + err.span_label(span, format!("`&{}mut` is only allowed in `const fn`", raw)); err }; if ccx.tcx.sess.teach(&err.get_code().unwrap()) { @@ -270,29 +277,6 @@ impl NonConstOp for MutBorrow { } } -// FIXME(ecstaticmorse): Unify this with `MutBorrow`. It has basically the same issues. -#[derive(Debug)] -pub struct MutAddressOf; -impl NonConstOp for MutAddressOf { - fn status_in_item(&self, ccx: &ConstCx<'_, '_>) -> Status { - // Forbid everywhere except in const fn with a feature gate - if ccx.const_kind() == hir::ConstContext::ConstFn { - Status::Unstable(sym::const_mut_refs) - } else { - Status::Forbidden - } - } - - fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> DiagnosticBuilder<'tcx> { - feature_err( - &ccx.tcx.sess.parse_sess, - sym::const_mut_refs, - span, - &format!("`&raw mut` is not allowed in {}s", ccx.const_kind()), - ) - } -} - #[derive(Debug)] pub struct MutDeref; impl NonConstOp for MutDeref { diff --git a/compiler/rustc_mir/src/transform/check_consts/validation.rs b/compiler/rustc_mir/src/transform/check_consts/validation.rs index 587b5b38128..4139b544998 100644 --- a/compiler/rustc_mir/src/transform/check_consts/validation.rs +++ b/compiler/rustc_mir/src/transform/check_consts/validation.rs @@ -292,7 +292,11 @@ impl Validator<'mir, 'tcx> { Status::Unstable(gate) if self.tcx.features().enabled(gate) => { let unstable_in_stable = self.ccx.is_const_stable_const_fn() - && !super::allow_internal_unstable(self.tcx, self.def_id().to_def_id(), gate); + && !super::rustc_allow_const_fn_unstable( + self.tcx, + self.def_id().to_def_id(), + gate, + ); if unstable_in_stable { emit_unstable_in_stable_error(self.ccx, span, gate); } @@ -525,14 +529,16 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { if !is_allowed { if let BorrowKind::Mut { .. } = kind { - self.check_op(ops::MutBorrow); + self.check_op(ops::MutBorrow(hir::BorrowKind::Ref)); } else { self.check_op(ops::CellBorrow); } } } - Rvalue::AddressOf(Mutability::Mut, _) => self.check_op(ops::MutAddressOf), + Rvalue::AddressOf(Mutability::Mut, _) => { + self.check_op(ops::MutBorrow(hir::BorrowKind::Raw)) + } Rvalue::Ref(_, BorrowKind::Shared | BorrowKind::Shallow, ref place) | Rvalue::AddressOf(Mutability::Not, ref place) => { @@ -805,7 +811,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { } // Calling an unstable function *always* requires that the corresponding gate - // be enabled, even if the function has `#[allow_internal_unstable(the_gate)]`. + // be enabled, even if the function has `#[rustc_allow_const_fn_unstable(the_gate)]`. if !tcx.features().declared_lib_features.iter().any(|&(sym, _)| sym == gate) { self.check_op(ops::FnCallUnstable(callee, Some(gate))); return; @@ -819,7 +825,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> { // Otherwise, we are something const-stable calling a const-unstable fn. - if super::allow_internal_unstable(tcx, caller, gate) { + if super::rustc_allow_const_fn_unstable(tcx, caller, gate) { return; } @@ -965,8 +971,8 @@ fn emit_unstable_in_stable_error(ccx: &ConstCx<'_, '_>, span: Span, gate: Symbol ) .span_suggestion( attr_span, - "otherwise `#[allow_internal_unstable]` can be used to bypass stability checks", - format!("#[allow_internal_unstable({})]\n", gate), + "otherwise `#[rustc_allow_const_fn_unstable]` can be used to bypass stability checks", + format!("#[rustc_allow_const_fn_unstable({})]\n", gate), Applicability::MaybeIncorrect, ) .emit(); diff --git a/compiler/rustc_mir/src/transform/check_unsafety.rs b/compiler/rustc_mir/src/transform/check_unsafety.rs index 7309a4129e4..3d68b862df2 100644 --- a/compiler/rustc_mir/src/transform/check_unsafety.rs +++ b/compiler/rustc_mir/src/transform/check_unsafety.rs @@ -204,6 +204,9 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { if let [] = proj_base { let decl = &self.body.local_decls[place.local]; if decl.internal { + // If the projection root is an artifical local that we introduced when + // desugaring `static`, give a more specific error message + // (avoid the general "raw pointer" clause below, that would only be confusing). if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info { if self.tcx.is_mutable_static(def_id) { self.require_unsafe( diff --git a/compiler/rustc_mir/src/transform/const_prop.rs b/compiler/rustc_mir/src/transform/const_prop.rs index 14b310cda93..d47e549b7be 100644 --- a/compiler/rustc_mir/src/transform/const_prop.rs +++ b/compiler/rustc_mir/src/transform/const_prop.rs @@ -9,7 +9,6 @@ use rustc_hir::def::DefKind; use rustc_hir::HirId; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; -use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::mir::visit::{ MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, }; @@ -28,9 +27,10 @@ use rustc_trait_selection::traits; use crate::const_eval::ConstEvalErr; use crate::interpret::{ - self, compile_time_machine, truncate, AllocId, Allocation, ConstValue, Frame, ImmTy, Immediate, - InterpCx, LocalState, LocalValue, MemPlace, Memory, MemoryKind, OpTy, Operand as InterpOperand, - PlaceTy, Pointer, ScalarMaybeUninit, StackPopCleanup, + self, compile_time_machine, truncate, AllocId, Allocation, ConstValue, CtfeValidationMode, + Frame, ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemPlace, Memory, + MemoryKind, OpTy, Operand as InterpOperand, PlaceTy, Pointer, Scalar, ScalarMaybeUninit, + StackPopCleanup, }; use crate::transform::MirPass; @@ -313,7 +313,7 @@ struct ConstPropagator<'mir, 'tcx> { param_env: ParamEnv<'tcx>, // FIXME(eddyb) avoid cloning these two fields more than once, // by accessing them through `ecx` instead. - source_scopes: IndexVec<SourceScope, SourceScopeData>, + source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, local_decls: IndexVec<Local, LocalDecl<'tcx>>, // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store // the last known `SourceInfo` here and just keep revisiting it. @@ -805,8 +805,9 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { value, vec![], // FIXME: is ref tracking too expensive? + // FIXME: what is the point of ref tracking if we do not even check the tracked refs? &mut interpret::RefTracking::empty(), - /*may_ref_to_static*/ true, + CtfeValidationMode::Regular, ) { trace!("validation error, attempt failed: {:?}", e); return; diff --git a/compiler/rustc_mir/src/transform/function_item_references.rs b/compiler/rustc_mir/src/transform/function_item_references.rs new file mode 100644 index 00000000000..61427422e4b --- /dev/null +++ b/compiler/rustc_mir/src/transform/function_item_references.rs @@ -0,0 +1,205 @@ +use rustc_errors::Applicability; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::*; +use rustc_middle::ty::{ + self, + subst::{GenericArgKind, Subst, SubstsRef}, + PredicateAtom, Ty, TyCtxt, TyS, +}; +use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; +use rustc_span::{symbol::sym, Span}; +use rustc_target::spec::abi::Abi; + +use crate::transform::MirPass; + +pub struct FunctionItemReferences; + +impl<'tcx> MirPass<'tcx> for FunctionItemReferences { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut checker = FunctionItemRefChecker { tcx, body }; + checker.visit_body(&body); + } +} + +struct FunctionItemRefChecker<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, +} + +impl<'a, 'tcx> Visitor<'tcx> for FunctionItemRefChecker<'a, 'tcx> { + /// Emits a lint for function reference arguments bound by `fmt::Pointer` or passed to + /// `transmute`. This only handles arguments in calls outside macro expansions to avoid double + /// counting function references formatted as pointers by macros. + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { + if let TerminatorKind::Call { + func, + args, + destination: _, + cleanup: _, + from_hir_call: _, + fn_span: _, + } = &terminator.kind + { + let source_info = *self.body.source_info(location); + // Only handle function calls outside macros + if !source_info.span.from_expansion() { + let func_ty = func.ty(self.body, self.tcx); + if let ty::FnDef(def_id, substs_ref) = *func_ty.kind() { + // Handle calls to `transmute` + if self.tcx.is_diagnostic_item(sym::transmute, def_id) { + let arg_ty = args[0].ty(self.body, self.tcx); + for generic_inner_ty in arg_ty.walk() { + if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { + if let Some(fn_id) = FunctionItemRefChecker::is_fn_ref(inner_ty) { + let ident = self.tcx.item_name(fn_id).to_ident_string(); + let span = self.nth_arg_span(&args, 0); + self.emit_lint(ident, fn_id, source_info, span); + } + } + } + } else { + self.check_bound_args(def_id, substs_ref, &args, source_info); + } + } + } + } + self.super_terminator(terminator, location); + } + /// Emits a lint for function references formatted with `fmt::Pointer::fmt` by macros. These + /// cases are handled as operands instead of call terminators to avoid any dependence on + /// unstable, internal formatting details like whether `fmt` is called directly or not. + fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { + let source_info = *self.body.source_info(location); + if source_info.span.from_expansion() { + let op_ty = operand.ty(self.body, self.tcx); + if let ty::FnDef(def_id, substs_ref) = *op_ty.kind() { + if self.tcx.is_diagnostic_item(sym::pointer_trait_fmt, def_id) { + let param_ty = substs_ref.type_at(0); + if let Some(fn_id) = FunctionItemRefChecker::is_fn_ref(param_ty) { + // The operand's ctxt wouldn't display the lint since it's inside a macro so + // we have to use the callsite's ctxt. + let callsite_ctxt = source_info.span.source_callsite().ctxt(); + let span = source_info.span.with_ctxt(callsite_ctxt); + let ident = self.tcx.item_name(fn_id).to_ident_string(); + self.emit_lint(ident, fn_id, source_info, span); + } + } + } + } + self.super_operand(operand, location); + } +} + +impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { + /// Emits a lint for function reference arguments bound by `fmt::Pointer` in calls to the + /// function defined by `def_id` with the substitutions `substs_ref`. + fn check_bound_args( + &self, + def_id: DefId, + substs_ref: SubstsRef<'tcx>, + args: &Vec<Operand<'tcx>>, + source_info: SourceInfo, + ) { + let param_env = self.tcx.param_env(def_id); + let bounds = param_env.caller_bounds(); + for bound in bounds { + if let Some(bound_ty) = self.is_pointer_trait(&bound.skip_binders()) { + // Get the argument types as they appear in the function signature. + let arg_defs = self.tcx.fn_sig(def_id).skip_binder().inputs(); + for (arg_num, arg_def) in arg_defs.iter().enumerate() { + // For all types reachable from the argument type in the fn sig + for generic_inner_ty in arg_def.walk() { + if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { + // If the inner type matches the type bound by `Pointer` + if TyS::same_type(inner_ty, bound_ty) { + // Do a substitution using the parameters from the callsite + let subst_ty = inner_ty.subst(self.tcx, substs_ref); + if let Some(fn_id) = FunctionItemRefChecker::is_fn_ref(subst_ty) { + let ident = self.tcx.item_name(fn_id).to_ident_string(); + let span = self.nth_arg_span(args, arg_num); + self.emit_lint(ident, fn_id, source_info, span); + } + } + } + } + } + } + } + } + /// If the given predicate is the trait `fmt::Pointer`, returns the bound parameter type. + fn is_pointer_trait(&self, bound: &PredicateAtom<'tcx>) -> Option<Ty<'tcx>> { + if let ty::PredicateAtom::Trait(predicate, _) = bound { + if self.tcx.is_diagnostic_item(sym::pointer_trait, predicate.def_id()) { + Some(predicate.trait_ref.self_ty()) + } else { + None + } + } else { + None + } + } + /// If a type is a reference or raw pointer to the anonymous type of a function definition, + /// returns that function's `DefId`. + fn is_fn_ref(ty: Ty<'tcx>) -> Option<DefId> { + let referent_ty = match ty.kind() { + ty::Ref(_, referent_ty, _) => Some(referent_ty), + ty::RawPtr(ty_and_mut) => Some(&ty_and_mut.ty), + _ => None, + }; + referent_ty + .map( + |ref_ty| { + if let ty::FnDef(def_id, _) = *ref_ty.kind() { Some(def_id) } else { None } + }, + ) + .unwrap_or(None) + } + fn nth_arg_span(&self, args: &Vec<Operand<'tcx>>, n: usize) -> Span { + match &args[n] { + Operand::Copy(place) | Operand::Move(place) => { + self.body.local_decls[place.local].source_info.span + } + Operand::Constant(constant) => constant.span, + } + } + fn emit_lint(&self, ident: String, fn_id: DefId, source_info: SourceInfo, span: Span) { + let lint_root = self.body.source_scopes[source_info.scope] + .local_data + .as_ref() + .assert_crate_local() + .lint_root; + let fn_sig = self.tcx.fn_sig(fn_id); + let unsafety = fn_sig.unsafety().prefix_str(); + let abi = match fn_sig.abi() { + Abi::Rust => String::from(""), + other_abi => { + let mut s = String::from("extern \""); + s.push_str(other_abi.name()); + s.push_str("\" "); + s + } + }; + let num_args = fn_sig.inputs().map_bound(|inputs| inputs.len()).skip_binder(); + let variadic = if fn_sig.c_variadic() { ", ..." } else { "" }; + let ret = if fn_sig.output().skip_binder().is_unit() { "" } else { " -> _" }; + self.tcx.struct_span_lint_hir(FUNCTION_ITEM_REFERENCES, lint_root, span, |lint| { + lint.build("taking a reference to a function item does not give a function pointer") + .span_suggestion( + span, + &format!("cast `{}` to obtain a function pointer", ident), + format!( + "{} as {}{}fn({}{}){}", + ident, + unsafety, + abi, + vec!["_"; num_args].join(", "), + variadic, + ret, + ), + Applicability::Unspecified, + ) + .emit(); + }); + } +} diff --git a/compiler/rustc_mir/src/transform/inline.rs b/compiler/rustc_mir/src/transform/inline.rs index 796ad6c9c29..944f41c61a2 100644 --- a/compiler/rustc_mir/src/transform/inline.rs +++ b/compiler/rustc_mir/src/transform/inline.rs @@ -1,13 +1,12 @@ //! Inlining pass for MIR functions use rustc_attr as attr; -use rustc_hir::def_id::DefId; use rustc_index::bit_set::BitSet; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::vec::Idx; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::subst::{Subst, SubstsRef}; +use rustc_middle::ty::subst::Subst; use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_target::spec::abi::Abi; @@ -15,6 +14,7 @@ use super::simplify::{remove_dead_blocks, CfgSimplifier}; use crate::transform::MirPass; use std::collections::VecDeque; use std::iter; +use std::ops::RangeFrom; const DEFAULT_THRESHOLD: usize = 50; const HINT_THRESHOLD: usize = 100; @@ -30,10 +30,9 @@ pub struct Inline; #[derive(Copy, Clone, Debug)] struct CallSite<'tcx> { - callee: DefId, - substs: SubstsRef<'tcx>, + callee: Instance<'tcx>, bb: BasicBlock, - location: SourceInfo, + source_info: SourceInfo, } impl<'tcx> MirPass<'tcx> for Inline { @@ -101,14 +100,20 @@ impl Inliner<'tcx> { local_change = false; while let Some(callsite) = callsites.pop_front() { debug!("checking whether to inline callsite {:?}", callsite); - if !self.tcx.is_mir_available(callsite.callee) { - debug!("checking whether to inline callsite {:?} - MIR unavailable", callsite); - continue; + + if let InstanceDef::Item(_) = callsite.callee.def { + if !self.tcx.is_mir_available(callsite.callee.def_id()) { + debug!( + "checking whether to inline callsite {:?} - MIR unavailable", + callsite, + ); + continue; + } } - let callee_body = if let Some(callee_def_id) = callsite.callee.as_local() { + let callee_body = if let Some(callee_def_id) = callsite.callee.def_id().as_local() { let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id); - // Avoid a cycle here by only using `optimized_mir` only if we have + // Avoid a cycle here by only using `instance_mir` only if we have // a lower `HirId` than the callee. This ensures that the callee will // not inline us. This trick only works without incremental compilation. // So don't do it if that is enabled. Also avoid inlining into generators, @@ -119,19 +124,21 @@ impl Inliner<'tcx> { && self_hir_id < callee_hir_id && caller_body.generator_kind.is_none() { - self.tcx.optimized_mir(callsite.callee) + self.tcx.instance_mir(callsite.callee.def) } else { continue; } } else { // This cannot result in a cycle since the callee MIR is from another crate // and is already optimized. - self.tcx.optimized_mir(callsite.callee) + self.tcx.instance_mir(callsite.callee.def) }; + let callee_body: &Body<'tcx> = &*callee_body; + let callee_body = if self.consider_optimizing(callsite, callee_body) { self.tcx.subst_and_normalize_erasing_regions( - &callsite.substs, + &callsite.callee.substs, self.param_env, callee_body, ) @@ -204,21 +211,16 @@ impl Inliner<'tcx> { // To resolve an instance its substs have to be fully normalized, so // we do this here. let normalized_substs = self.tcx.normalize_erasing_regions(self.param_env, substs); - let instance = + let callee = Instance::resolve(self.tcx, self.param_env, callee_def_id, normalized_substs) .ok() .flatten()?; - if let InstanceDef::Virtual(..) = instance.def { + if let InstanceDef::Virtual(..) | InstanceDef::Intrinsic(_) = callee.def { return None; } - return Some(CallSite { - callee: instance.def_id(), - substs: instance.substs, - bb, - location: terminator.source_info, - }); + return Some(CallSite { callee, bb, source_info: terminator.source_info }); } } @@ -243,12 +245,7 @@ impl Inliner<'tcx> { return false; } - let codegen_fn_attrs = tcx.codegen_fn_attrs(callsite.callee); - - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::TRACK_CALLER) { - debug!("`#[track_caller]` present - not inlining"); - return false; - } + let codegen_fn_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); let self_features = &self.codegen_fn_attrs.target_features; let callee_features = &codegen_fn_attrs.target_features; @@ -282,8 +279,8 @@ impl Inliner<'tcx> { // Only inline local functions if they would be eligible for cross-crate // inlining. This is to ensure that the final crate doesn't have MIR that // reference unexported symbols - if callsite.callee.is_local() { - if callsite.substs.non_erasable_generics().count() == 0 && !hinted { + if callsite.callee.def_id().is_local() { + if callsite.callee.substs.non_erasable_generics().count() == 0 && !hinted { debug!(" callee is an exported function - not inlining"); return false; } @@ -336,7 +333,7 @@ impl Inliner<'tcx> { work_list.push(target); // If the place doesn't actually need dropping, treat it like // a regular goto. - let ty = place.ty(callee_body, tcx).subst(tcx, callsite.substs).ty; + let ty = place.ty(callee_body, tcx).subst(tcx, callsite.callee.substs).ty; if ty.needs_drop(tcx, self.param_env) { cost += CALL_PENALTY; if let Some(unwind) = unwind { @@ -399,7 +396,7 @@ impl Inliner<'tcx> { for v in callee_body.vars_and_temps_iter() { let v = &callee_body.local_decls[v]; - let ty = v.ty.subst(tcx, callsite.substs); + let ty = v.ty.subst(tcx, callsite.callee.substs); // Cost of the var is the size in machine-words, if we know // it. if let Some(size) = type_size_of(tcx, self.param_env, ty) { @@ -435,36 +432,6 @@ impl Inliner<'tcx> { TerminatorKind::Call { args, destination: Some(destination), cleanup, .. } => { debug!("inlined {:?} into {:?}", callsite.callee, caller_body.source); - let mut local_map = IndexVec::with_capacity(callee_body.local_decls.len()); - let mut scope_map = IndexVec::with_capacity(callee_body.source_scopes.len()); - - for mut scope in callee_body.source_scopes.iter().cloned() { - if scope.parent_scope.is_none() { - scope.parent_scope = Some(callsite.location.scope); - // FIXME(eddyb) is this really needed? - // (also note that it's always overwritten below) - scope.span = callee_body.span; - } - - // FIXME(eddyb) this doesn't seem right at all. - // The inlined source scopes should probably be annotated as - // such, but also contain all of the original information. - scope.span = callsite.location.span; - - let idx = caller_body.source_scopes.push(scope); - scope_map.push(idx); - } - - for loc in callee_body.vars_and_temps_iter() { - let mut local = callee_body.local_decls[loc].clone(); - - local.source_info.scope = scope_map[local.source_info.scope]; - local.source_info.span = callsite.location.span; - - let idx = caller_body.local_decls.push(local); - local_map.push(idx); - } - // If the call is something like `a[*i] = f(i)`, where // `i : &mut usize`, then just duplicating the `a[*i]` // Place could result in two different locations if `f` @@ -491,13 +458,13 @@ impl Inliner<'tcx> { let ty = dest.ty(caller_body, self.tcx); - let temp = LocalDecl::new(ty, callsite.location.span); + let temp = LocalDecl::new(ty, callsite.source_info.span); let tmp = caller_body.local_decls.push(temp); let tmp = Place::from(tmp); let stmt = Statement { - source_info: callsite.location, + source_info: callsite.source_info, kind: StatementKind::Assign(box (tmp, dest)), }; caller_body[callsite.bb].statements.push(stmt); @@ -511,12 +478,11 @@ impl Inliner<'tcx> { // Copy the arguments if needed. let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, return_block); - let bb_len = caller_body.basic_blocks().len(); let mut integrator = Integrator { - block_idx: bb_len, args: &args, - local_map, - scope_map, + new_locals: Local::new(caller_body.local_decls.len()).., + new_scopes: SourceScope::new(caller_body.source_scopes.len()).., + new_blocks: BasicBlock::new(caller_body.basic_blocks().len()).., destination: dest, return_block, cleanup_block: cleanup, @@ -524,22 +490,51 @@ impl Inliner<'tcx> { tcx: self.tcx, }; - for mut var_debug_info in callee_body.var_debug_info.drain(..) { - integrator.visit_var_debug_info(&mut var_debug_info); - caller_body.var_debug_info.push(var_debug_info); - } + // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones + // (or existing ones, in a few special cases) in the caller. + integrator.visit_body(&mut callee_body); - for (bb, mut block) in callee_body.basic_blocks_mut().drain_enumerated(..) { - integrator.visit_basic_block_data(bb, &mut block); - caller_body.basic_blocks_mut().push(block); + for scope in &mut callee_body.source_scopes { + // FIXME(eddyb) move this into a `fn visit_scope_data` in `Integrator`. + if scope.parent_scope.is_none() { + let callsite_scope = &caller_body.source_scopes[callsite.source_info.scope]; + + // Attach the outermost callee scope as a child of the callsite + // scope, via the `parent_scope` and `inlined_parent_scope` chains. + scope.parent_scope = Some(callsite.source_info.scope); + assert_eq!(scope.inlined_parent_scope, None); + scope.inlined_parent_scope = if callsite_scope.inlined.is_some() { + Some(callsite.source_info.scope) + } else { + callsite_scope.inlined_parent_scope + }; + + // Mark the outermost callee scope as an inlined one. + assert_eq!(scope.inlined, None); + scope.inlined = Some((callsite.callee, callsite.source_info.span)); + } else if scope.inlined_parent_scope.is_none() { + // Make it easy to find the scope with `inlined` set above. + scope.inlined_parent_scope = + Some(integrator.map_scope(OUTERMOST_SOURCE_SCOPE)); + } } - let terminator = Terminator { - source_info: callsite.location, - kind: TerminatorKind::Goto { target: BasicBlock::new(bb_len) }, - }; + // Insert all of the (mapped) parts of the callee body into the caller. + caller_body.local_decls.extend( + // FIXME(eddyb) make `Range<Local>` iterable so that we can use + // `callee_body.local_decls.drain(callee_body.vars_and_temps())` + callee_body + .vars_and_temps_iter() + .map(|local| callee_body.local_decls[local].clone()), + ); + caller_body.source_scopes.extend(callee_body.source_scopes.drain(..)); + caller_body.var_debug_info.extend(callee_body.var_debug_info.drain(..)); + caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..)); - caller_body[callsite.bb].terminator = Some(terminator); + caller_body[callsite.bb].terminator = Some(Terminator { + source_info: callsite.source_info, + kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) }, + }); true } @@ -583,7 +578,9 @@ impl Inliner<'tcx> { // tmp2 = tuple_tmp.2 // // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`. - if tcx.is_closure(callsite.callee) { + // FIXME(eddyb) make this check for `"rust-call"` ABI combined with + // `callee_body.spread_arg == None`, instead of special-casing closures. + if tcx.is_closure(callsite.callee.def_id()) { let mut args = args.into_iter(); let self_ = self.create_temp_if_necessary( args.next().unwrap(), @@ -654,20 +651,23 @@ impl Inliner<'tcx> { let ty = arg.ty(caller_body, self.tcx); - let arg_tmp = LocalDecl::new(ty, callsite.location.span); + let arg_tmp = LocalDecl::new(ty, callsite.source_info.span); let arg_tmp = caller_body.local_decls.push(arg_tmp); caller_body[callsite.bb].statements.push(Statement { - source_info: callsite.location, + source_info: callsite.source_info, kind: StatementKind::StorageLive(arg_tmp), }); caller_body[callsite.bb].statements.push(Statement { - source_info: callsite.location, + source_info: callsite.source_info, kind: StatementKind::Assign(box (Place::from(arg_tmp), arg)), }); caller_body[return_block].statements.insert( 0, - Statement { source_info: callsite.location, kind: StatementKind::StorageDead(arg_tmp) }, + Statement { + source_info: callsite.source_info, + kind: StatementKind::StorageDead(arg_tmp), + }, ); arg_tmp @@ -690,10 +690,10 @@ fn type_size_of<'tcx>( * stuff. */ struct Integrator<'a, 'tcx> { - block_idx: usize, args: &'a [Local], - local_map: IndexVec<Local, Local>, - scope_map: IndexVec<SourceScope, SourceScope>, + new_locals: RangeFrom<Local>, + new_scopes: RangeFrom<SourceScope>, + new_blocks: RangeFrom<BasicBlock>, destination: Place<'tcx>, return_block: BasicBlock, cleanup_block: Option<BasicBlock>, @@ -702,23 +702,31 @@ struct Integrator<'a, 'tcx> { } impl<'a, 'tcx> Integrator<'a, 'tcx> { - fn update_target(&self, tgt: BasicBlock) -> BasicBlock { - let new = BasicBlock::new(tgt.index() + self.block_idx); - debug!("updating target `{:?}`, new: `{:?}`", tgt, new); + fn map_local(&self, local: Local) -> Local { + let new = if local == RETURN_PLACE { + self.destination.local + } else { + let idx = local.index() - 1; + if idx < self.args.len() { + self.args[idx] + } else { + Local::new(self.new_locals.start.index() + (idx - self.args.len())) + } + }; + debug!("mapping local `{:?}` to `{:?}`", local, new); new } - fn make_integrate_local(&self, local: Local) -> Local { - if local == RETURN_PLACE { - return self.destination.local; - } - - let idx = local.index() - 1; - if idx < self.args.len() { - return self.args[idx]; - } + fn map_scope(&self, scope: SourceScope) -> SourceScope { + let new = SourceScope::new(self.new_scopes.start.index() + scope.index()); + debug!("mapping scope `{:?}` to `{:?}`", scope, new); + new + } - self.local_map[Local::new(idx - self.args.len())] + fn map_block(&self, block: BasicBlock) -> BasicBlock { + let new = BasicBlock::new(self.new_blocks.start.index() + block.index()); + debug!("mapping block `{:?}` to `{:?}`", block, new); + new } } @@ -728,7 +736,11 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } fn visit_local(&mut self, local: &mut Local, _ctxt: PlaceContext, _location: Location) { - *local = self.make_integrate_local(*local); + *local = self.map_local(*local); + } + + fn visit_source_scope(&mut self, scope: &mut SourceScope) { + *scope = self.map_scope(*scope); } fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { @@ -772,18 +784,18 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { match terminator.kind { TerminatorKind::GeneratorDrop | TerminatorKind::Yield { .. } => bug!(), TerminatorKind::Goto { ref mut target } => { - *target = self.update_target(*target); + *target = self.map_block(*target); } TerminatorKind::SwitchInt { ref mut targets, .. } => { for tgt in targets.all_targets_mut() { - *tgt = self.update_target(*tgt); + *tgt = self.map_block(*tgt); } } TerminatorKind::Drop { ref mut target, ref mut unwind, .. } | TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => { - *target = self.update_target(*target); + *target = self.map_block(*target); if let Some(tgt) = *unwind { - *unwind = Some(self.update_target(tgt)); + *unwind = Some(self.map_block(tgt)); } else if !self.in_cleanup_block { // Unless this drop is in a cleanup block, add an unwind edge to // the original call's cleanup block @@ -792,10 +804,10 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } TerminatorKind::Call { ref mut destination, ref mut cleanup, .. } => { if let Some((_, ref mut tgt)) = *destination { - *tgt = self.update_target(*tgt); + *tgt = self.map_block(*tgt); } if let Some(tgt) = *cleanup { - *cleanup = Some(self.update_target(tgt)); + *cleanup = Some(self.map_block(tgt)); } else if !self.in_cleanup_block { // Unless this call is in a cleanup block, add an unwind edge to // the original call's cleanup block @@ -803,9 +815,9 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } } TerminatorKind::Assert { ref mut target, ref mut cleanup, .. } => { - *target = self.update_target(*target); + *target = self.map_block(*target); if let Some(tgt) = *cleanup { - *cleanup = Some(self.update_target(tgt)); + *cleanup = Some(self.map_block(tgt)); } else if !self.in_cleanup_block { // Unless this assert is in a cleanup block, add an unwind edge to // the original call's cleanup block @@ -823,8 +835,8 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { TerminatorKind::Abort => {} TerminatorKind::Unreachable => {} TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => { - *real_target = self.update_target(*real_target); - *imaginary_target = self.update_target(*imaginary_target); + *real_target = self.map_block(*real_target); + *imaginary_target = self.map_block(*imaginary_target); } TerminatorKind::FalseUnwind { real_target: _, unwind: _ } => // see the ordering of passes in the optimized_mir query. @@ -833,13 +845,9 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { } TerminatorKind::InlineAsm { ref mut destination, .. } => { if let Some(ref mut tgt) = *destination { - *tgt = self.update_target(*tgt); + *tgt = self.map_block(*tgt); } } } } - - fn visit_source_scope(&mut self, scope: &mut SourceScope) { - *scope = self.scope_map[*scope]; - } } diff --git a/compiler/rustc_mir/src/transform/instcombine.rs b/compiler/rustc_mir/src/transform/instcombine.rs index 1a8e281d417..59b7db24319 100644 --- a/compiler/rustc_mir/src/transform/instcombine.rs +++ b/compiler/rustc_mir/src/transform/instcombine.rs @@ -119,6 +119,11 @@ impl OptimizationFinder<'b, 'tcx> { } fn find_deref_of_address(&mut self, rvalue: &Rvalue<'tcx>, location: Location) -> Option<()> { + // FIXME(#78192): This optimization can result in unsoundness. + if !self.tcx.sess.opts.debugging_opts.unsound_mir_opts { + return None; + } + // Look for the sequence // // _2 = &_1; @@ -132,6 +137,8 @@ impl OptimizationFinder<'b, 'tcx> { _ => None, }?; + let mut dead_locals_seen = vec![]; + let stmt_index = location.statement_index; // Look behind for statement that assigns the local from a address of operator. // 6 is chosen as a heuristic determined by seeing the number of times @@ -155,6 +162,11 @@ impl OptimizationFinder<'b, 'tcx> { BorrowKind::Shared, place_taken_address_of, ) => { + // Make sure that the place has not been marked dead + if dead_locals_seen.contains(&place_taken_address_of.local) { + return None; + } + self.optimizations .unneeded_deref .insert(location, *place_taken_address_of); @@ -173,13 +185,19 @@ impl OptimizationFinder<'b, 'tcx> { // Inline asm can do anything, so bail out of the optimization. rustc_middle::mir::StatementKind::LlvmInlineAsm(_) => return None, + // Remember `StorageDead`s, as the local being marked dead could be the + // place RHS we are looking for, in which case we need to abort to avoid UB + // using an uninitialized place + rustc_middle::mir::StatementKind::StorageDead(dead) => { + dead_locals_seen.push(*dead) + } + // Check that `local_being_deref` is not being used in a mutating way which can cause misoptimization. rustc_middle::mir::StatementKind::Assign(box (_, _)) | rustc_middle::mir::StatementKind::Coverage(_) | rustc_middle::mir::StatementKind::Nop | rustc_middle::mir::StatementKind::FakeRead(_, _) | rustc_middle::mir::StatementKind::StorageLive(_) - | rustc_middle::mir::StatementKind::StorageDead(_) | rustc_middle::mir::StatementKind::Retag(_, _) | rustc_middle::mir::StatementKind::AscribeUserType(_, _) | rustc_middle::mir::StatementKind::SetDiscriminant { .. } => { diff --git a/compiler/rustc_mir/src/transform/match_branches.rs b/compiler/rustc_mir/src/transform/match_branches.rs index 8b2d6b09aa8..06690dcbf6e 100644 --- a/compiler/rustc_mir/src/transform/match_branches.rs +++ b/compiler/rustc_mir/src/transform/match_branches.rs @@ -38,12 +38,16 @@ pub struct MatchBranchSimplification; impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + if tcx.sess.opts.debugging_opts.mir_opt_level <= 1 { + return; + } + let param_env = tcx.param_env(body.source.def_id()); - let bbs = body.basic_blocks_mut(); + let (bbs, local_decls) = body.basic_blocks_and_local_decls_mut(); 'outer: for bb_idx in bbs.indices() { let (discr, val, switch_ty, first, second) = match bbs[bb_idx].terminator().kind { TerminatorKind::SwitchInt { - discr: Operand::Copy(ref place) | Operand::Move(ref place), + discr: ref discr @ (Operand::Copy(_) | Operand::Move(_)), switch_ty, ref targets, .. @@ -52,7 +56,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { if target == targets.otherwise() { continue; } - (place, value, switch_ty, target, targets.otherwise()) + (discr, value, switch_ty, target, targets.otherwise()) } // Only optimize switch int statements _ => continue, @@ -92,6 +96,10 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { // Take ownership of items now that we know we can optimize. let discr = discr.clone(); + // Introduce a temporary for the discriminant value. + let source_info = bbs[bb_idx].terminator().source_info; + let discr_local = local_decls.push(LocalDecl::new(switch_ty, source_info.span)); + // We already checked that first and second are different blocks, // and bb_idx has a different terminator from both of them. let (from, first, second) = bbs.pick3_mut(bb_idx, first, second); @@ -120,7 +128,11 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { rustc_span::DUMMY_SP, ); let op = if f_b { BinOp::Eq } else { BinOp::Ne }; - let rhs = Rvalue::BinaryOp(op, Operand::Copy(discr.clone()), const_cmp); + let rhs = Rvalue::BinaryOp( + op, + Operand::Copy(Place::from(discr_local)), + const_cmp, + ); Statement { source_info: f.source_info, kind: StatementKind::Assign(box (*lhs, rhs)), @@ -131,7 +143,16 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { _ => unreachable!(), } }); + + from.statements + .push(Statement { source_info, kind: StatementKind::StorageLive(discr_local) }); + from.statements.push(Statement { + source_info, + kind: StatementKind::Assign(box (Place::from(discr_local), Rvalue::Use(discr))), + }); from.statements.extend(new_stmts); + from.statements + .push(Statement { source_info, kind: StatementKind::StorageDead(discr_local) }); from.terminator_mut().kind = first.terminator().kind.clone(); } } diff --git a/compiler/rustc_mir/src/transform/mod.rs b/compiler/rustc_mir/src/transform/mod.rs index 20b8c90a9dc..89db6bb13ca 100644 --- a/compiler/rustc_mir/src/transform/mod.rs +++ b/compiler/rustc_mir/src/transform/mod.rs @@ -27,6 +27,7 @@ pub mod dest_prop; pub mod dump_mir; pub mod early_otherwise_branch; pub mod elaborate_drops; +pub mod function_item_references; pub mod generator; pub mod inline; pub mod instcombine; @@ -266,6 +267,7 @@ fn mir_const<'tcx>( // MIR-level lints. &check_packed_ref::CheckPackedRef, &check_const_item_mutation::CheckConstItemMutation, + &function_item_references::FunctionItemReferences, // What we need to do constant evaluation. &simplify::SimplifyCfg::new("initial"), &rustc_peek::SanityCheck, diff --git a/compiler/rustc_mir/src/transform/nrvo.rs b/compiler/rustc_mir/src/transform/nrvo.rs index 7e05d66074b..45b906bf542 100644 --- a/compiler/rustc_mir/src/transform/nrvo.rs +++ b/compiler/rustc_mir/src/transform/nrvo.rs @@ -1,3 +1,5 @@ +//! See the docs for [`RenameReturnPlace`]. + use rustc_hir::Mutability; use rustc_index::bit_set::HybridBitSet; use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor}; diff --git a/compiler/rustc_mir/src/transform/promote_consts.rs b/compiler/rustc_mir/src/transform/promote_consts.rs index 7373abcc820..927aae82a36 100644 --- a/compiler/rustc_mir/src/transform/promote_consts.rs +++ b/compiler/rustc_mir/src/transform/promote_consts.rs @@ -301,17 +301,6 @@ impl std::ops::Deref for Validator<'a, 'tcx> { struct Unpromotable; impl<'tcx> Validator<'_, 'tcx> { - /// Determines if this code could be executed at runtime and thus is subject to codegen. - /// That means even unused constants need to be evaluated. - /// - /// `const_kind` should not be used in this file other than through this method! - fn maybe_runtime(&self) -> bool { - match self.const_kind { - None | Some(hir::ConstContext::ConstFn) => true, - Some(hir::ConstContext::Static(_) | hir::ConstContext::Const) => false, - } - } - fn validate_candidate(&self, candidate: Candidate) -> Result<(), Unpromotable> { match candidate { Candidate::Ref(loc) => { @@ -562,14 +551,12 @@ impl<'tcx> Validator<'_, 'tcx> { } ProjectionElem::Field(..) => { - if self.maybe_runtime() { - let base_ty = - Place::ty_from(place.local, proj_base, self.body, self.tcx).ty; - if let Some(def) = base_ty.ty_adt_def() { - // No promotion of union field accesses. - if def.is_union() { - return Err(Unpromotable); - } + let base_ty = + Place::ty_from(place.local, proj_base, self.body, self.tcx).ty; + if let Some(def) = base_ty.ty_adt_def() { + // No promotion of union field accesses. + if def.is_union() { + return Err(Unpromotable); } } } @@ -751,7 +738,14 @@ impl<'tcx> Validator<'_, 'tcx> { ) -> Result<(), Unpromotable> { let fn_ty = callee.ty(self.body, self.tcx); - if !self.explicit && self.maybe_runtime() { + // When doing explicit promotion and inside const/static items, we promote all (eligible) function calls. + // Everywhere else, we require `#[rustc_promotable]` on the callee. + let promote_all_const_fn = self.explicit + || matches!( + self.const_kind, + Some(hir::ConstContext::Static(_) | hir::ConstContext::Const) + ); + if !promote_all_const_fn { if let ty::FnDef(def_id, _) = *fn_ty.kind() { // Never promote runtime `const fn` calls of // functions without `#[rustc_promotable]`. @@ -1176,7 +1170,7 @@ pub fn promote_candidates<'tcx>( let mut scope = body.source_scopes[candidate.source_info(body).scope].clone(); scope.parent_scope = None; - let mut promoted = Body::new( + let promoted = Body::new( body.source, // `promoted` gets filled in below IndexVec::new(), IndexVec::from_elem_n(scope, 1), @@ -1187,7 +1181,6 @@ pub fn promote_candidates<'tcx>( body.span, body.generator_kind, ); - promoted.ignore_interior_mut_in_const_validation = true; let promoter = Promoter { promoted, diff --git a/compiler/rustc_mir/src/transform/simplify.rs b/compiler/rustc_mir/src/transform/simplify.rs index f0c87bcf513..b7c9a3a8688 100644 --- a/compiler/rustc_mir/src/transform/simplify.rs +++ b/compiler/rustc_mir/src/transform/simplify.rs @@ -35,6 +35,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use smallvec::SmallVec; use std::borrow::Cow; +use std::convert::TryInto; pub struct SimplifyCfg { label: String, @@ -322,32 +323,17 @@ impl<'tcx> MirPass<'tcx> for SimplifyLocals { trace!("running SimplifyLocals on {:?}", body.source); // First, we're going to get a count of *actual* uses for every `Local`. - // Take a look at `DeclMarker::visit_local()` to see exactly what is ignored. - let mut used_locals = { - let mut marker = DeclMarker::new(body); - marker.visit_body(&body); - - marker.local_counts - }; - - let arg_count = body.arg_count; + let mut used_locals = UsedLocals::new(body); // Next, we're going to remove any `Local` with zero actual uses. When we remove those // `Locals`, we're also going to subtract any uses of other `Locals` from the `used_locals` // count. For example, if we removed `_2 = discriminant(_1)`, then we'll subtract one from // `use_counts[_1]`. That in turn might make `_1` unused, so we loop until we hit a // fixedpoint where there are no more unused locals. - loop { - let mut remove_statements = RemoveStatements::new(&mut used_locals, arg_count, tcx); - remove_statements.visit_body(body); - - if !remove_statements.modified { - break; - } - } + remove_unused_definitions(&mut used_locals, body); // Finally, we'll actually do the work of shrinking `body.local_decls` and remapping the `Local`s. - let map = make_local_map(&mut body.local_decls, used_locals, arg_count); + let map = make_local_map(&mut body.local_decls, &used_locals); // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { @@ -363,14 +349,14 @@ impl<'tcx> MirPass<'tcx> for SimplifyLocals { /// Construct the mapping while swapping out unused stuff out from the `vec`. fn make_local_map<V>( local_decls: &mut IndexVec<Local, V>, - used_locals: IndexVec<Local, usize>, - arg_count: usize, + used_locals: &UsedLocals, ) -> IndexVec<Local, Option<Local>> { let mut map: IndexVec<Local, Option<Local>> = IndexVec::from_elem(None, &*local_decls); let mut used = Local::new(0); - for (alive_index, count) in used_locals.iter_enumerated() { - // The `RETURN_PLACE` and arguments are always live. - if alive_index.as_usize() > arg_count && *count == 0 { + + for alive_index in local_decls.indices() { + // `is_used` treats the `RETURN_PLACE` and arguments as used. + if !used_locals.is_used(alive_index) { continue; } @@ -384,149 +370,130 @@ fn make_local_map<V>( map } -struct DeclMarker<'a, 'tcx> { - pub local_counts: IndexVec<Local, usize>, - pub body: &'a Body<'tcx>, +/// Keeps track of used & unused locals. +struct UsedLocals { + increment: bool, + arg_count: u32, + use_count: IndexVec<Local, u32>, } -impl<'a, 'tcx> DeclMarker<'a, 'tcx> { - pub fn new(body: &'a Body<'tcx>) -> Self { - Self { local_counts: IndexVec::from_elem(0, &body.local_decls), body } +impl UsedLocals { + /// Determines which locals are used & unused in the given body. + fn new(body: &Body<'_>) -> Self { + let mut this = Self { + increment: true, + arg_count: body.arg_count.try_into().unwrap(), + use_count: IndexVec::from_elem(0, &body.local_decls), + }; + this.visit_body(body); + this } -} -impl<'a, 'tcx> Visitor<'tcx> for DeclMarker<'a, 'tcx> { - fn visit_local(&mut self, local: &Local, ctx: PlaceContext, location: Location) { - // Ignore storage markers altogether, they get removed along with their otherwise unused - // decls. - // FIXME: Extend this to all non-uses. - if ctx.is_storage_marker() { - return; - } + /// Checks if local is used. + /// + /// Return place and arguments are always considered used. + fn is_used(&self, local: Local) -> bool { + trace!("is_used({:?}): use_count: {:?}", local, self.use_count[local]); + local.as_u32() <= self.arg_count || self.use_count[local] != 0 + } - // Ignore stores of constants because `ConstProp` and `CopyProp` can remove uses of many - // of these locals. However, if the local is still needed, then it will be referenced in - // another place and we'll mark it as being used there. - if ctx == PlaceContext::MutatingUse(MutatingUseContext::Store) - || ctx == PlaceContext::MutatingUse(MutatingUseContext::Projection) - { - let block = &self.body.basic_blocks()[location.block]; - if location.statement_index != block.statements.len() { - let stmt = &block.statements[location.statement_index]; - - if let StatementKind::Assign(box (dest, rvalue)) = &stmt.kind { - if !dest.is_indirect() && dest.local == *local { - let can_skip = match rvalue { - Rvalue::Use(_) - | Rvalue::Discriminant(_) - | Rvalue::BinaryOp(_, _, _) - | Rvalue::CheckedBinaryOp(_, _, _) - | Rvalue::Repeat(_, _) - | Rvalue::AddressOf(_, _) - | Rvalue::Len(_) - | Rvalue::UnaryOp(_, _) - | Rvalue::Aggregate(_, _) => true, - - _ => false, - }; - - if can_skip { - trace!("skipping store of {:?} to {:?}", rvalue, dest); - return; - } - } - } - } - } + /// Updates the use counts to reflect the removal of given statement. + fn statement_removed(&mut self, statement: &Statement<'tcx>) { + self.increment = false; - self.local_counts[*local] += 1; + // The location of the statement is irrelevant. + let location = Location { block: START_BLOCK, statement_index: 0 }; + self.visit_statement(statement, location); } -} - -struct StatementDeclMarker<'a, 'tcx> { - used_locals: &'a mut IndexVec<Local, usize>, - statement: &'a Statement<'tcx>, -} -impl<'a, 'tcx> StatementDeclMarker<'a, 'tcx> { - pub fn new( - used_locals: &'a mut IndexVec<Local, usize>, - statement: &'a Statement<'tcx>, - ) -> Self { - Self { used_locals, statement } + /// Visits a left-hand side of an assignment. + fn visit_lhs(&mut self, place: &Place<'tcx>, location: Location) { + if place.is_indirect() { + // A use, not a definition. + self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location); + } else { + // A definition. Although, it still might use other locals for indexing. + self.super_projection( + place.local, + &place.projection, + PlaceContext::MutatingUse(MutatingUseContext::Projection), + location, + ); + } } } -impl<'a, 'tcx> Visitor<'tcx> for StatementDeclMarker<'a, 'tcx> { - fn visit_local(&mut self, local: &Local, context: PlaceContext, _location: Location) { - // Skip the lvalue for assignments - if let StatementKind::Assign(box (p, _)) = self.statement.kind { - if p.local == *local && context.is_place_assignment() { - return; +impl Visitor<'_> for UsedLocals { + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + match statement.kind { + StatementKind::LlvmInlineAsm(..) + | StatementKind::Retag(..) + | StatementKind::Coverage(..) + | StatementKind::FakeRead(..) + | StatementKind::AscribeUserType(..) => { + self.super_statement(statement, location); } - } - let use_count = &mut self.used_locals[*local]; - // If this is the local we're removing... - if *use_count != 0 { - *use_count -= 1; - } - } -} + StatementKind::Nop => {} -struct RemoveStatements<'a, 'tcx> { - used_locals: &'a mut IndexVec<Local, usize>, - arg_count: usize, - tcx: TyCtxt<'tcx>, - modified: bool, -} + StatementKind::StorageLive(_local) | StatementKind::StorageDead(_local) => {} -impl<'a, 'tcx> RemoveStatements<'a, 'tcx> { - fn new( - used_locals: &'a mut IndexVec<Local, usize>, - arg_count: usize, - tcx: TyCtxt<'tcx>, - ) -> Self { - Self { used_locals, arg_count, tcx, modified: false } - } + StatementKind::Assign(box (ref place, ref rvalue)) => { + self.visit_lhs(place, location); + self.visit_rvalue(rvalue, location); + } - fn keep_local(&self, l: Local) -> bool { - trace!("keep_local({:?}): count: {:?}", l, self.used_locals[l]); - l.as_usize() <= self.arg_count || self.used_locals[l] != 0 + StatementKind::SetDiscriminant { ref place, variant_index: _ } => { + self.visit_lhs(place, location); + } + } } -} -impl<'a, 'tcx> MutVisitor<'tcx> for RemoveStatements<'a, 'tcx> { - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx + fn visit_local(&mut self, local: &Local, _ctx: PlaceContext, _location: Location) { + if self.increment { + self.use_count[*local] += 1; + } else { + assert_ne!(self.use_count[*local], 0); + self.use_count[*local] -= 1; + } } +} - fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { - // Remove unnecessary StorageLive and StorageDead annotations. - let mut i = 0usize; - data.statements.retain(|stmt| { - let keep = match &stmt.kind { - StatementKind::StorageLive(l) | StatementKind::StorageDead(l) => { - self.keep_local(*l) - } - StatementKind::Assign(box (place, _)) => self.keep_local(place.local), - _ => true, - }; - - if !keep { - trace!("removing statement {:?}", stmt); - self.modified = true; - - let mut visitor = StatementDeclMarker::new(self.used_locals, stmt); - visitor.visit_statement(stmt, Location { block, statement_index: i }); - } +/// Removes unused definitions. Updates the used locals to reflect the changes made. +fn remove_unused_definitions<'a, 'tcx>(used_locals: &'a mut UsedLocals, body: &mut Body<'tcx>) { + // The use counts are updated as we remove the statements. A local might become unused + // during the retain operation, leading to a temporary inconsistency (storage statements or + // definitions referencing the local might remain). For correctness it is crucial that this + // computation reaches a fixed point. + + let mut modified = true; + while modified { + modified = false; + + for data in body.basic_blocks_mut() { + // Remove unnecessary StorageLive and StorageDead annotations. + data.statements.retain(|statement| { + let keep = match &statement.kind { + StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + used_locals.is_used(*local) + } + StatementKind::Assign(box (place, _)) => used_locals.is_used(place.local), - i += 1; + StatementKind::SetDiscriminant { ref place, .. } => { + used_locals.is_used(place.local) + } + _ => true, + }; - keep - }); + if !keep { + trace!("removing statement {:?}", statement); + modified = true; + used_locals.statement_removed(statement); + } - self.super_basic_block_data(block, data); + keep + }); + } } } diff --git a/compiler/rustc_mir/src/transform/simplify_branches.rs b/compiler/rustc_mir/src/transform/simplify_branches.rs index 5f63c03993d..a9a45e61a38 100644 --- a/compiler/rustc_mir/src/transform/simplify_branches.rs +++ b/compiler/rustc_mir/src/transform/simplify_branches.rs @@ -49,9 +49,10 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranches { } TerminatorKind::Assert { target, cond: Operand::Constant(ref c), expected, .. - } if (c.literal.try_eval_bool(tcx, param_env) == Some(true)) == expected => { - TerminatorKind::Goto { target } - } + } => match c.literal.try_eval_bool(tcx, param_env) { + Some(v) if v == expected => TerminatorKind::Goto { target }, + _ => continue, + }, TerminatorKind::FalseEdge { real_target, .. } => { TerminatorKind::Goto { target: real_target } } diff --git a/compiler/rustc_mir/src/transform/validate.rs b/compiler/rustc_mir/src/transform/validate.rs index beffffa727e..1868d3b47f2 100644 --- a/compiler/rustc_mir/src/transform/validate.rs +++ b/compiler/rustc_mir/src/transform/validate.rs @@ -5,13 +5,17 @@ use crate::dataflow::{Analysis, ResultsCursor}; use crate::util::storage::AlwaysLiveLocals; use super::MirPass; +use rustc_index::bit_set::BitSet; +use rustc_middle::mir::interpret::Scalar; +use rustc_middle::mir::traversal; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::{ - AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPhase, Operand, Rvalue, - SourceScope, Statement, StatementKind, Terminator, TerminatorKind, VarDebugInfo, + AggregateKind, BasicBlock, Body, BorrowKind, Local, Location, MirPhase, Operand, PlaceRef, + Rvalue, SourceScope, Statement, StatementKind, Terminator, TerminatorKind, VarDebugInfo, }; use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation}; use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt}; +use rustc_target::abi::Size; #[derive(Copy, Clone, Debug)] enum EdgeKind { @@ -42,8 +46,17 @@ impl<'tcx> MirPass<'tcx> for Validator { .iterate_to_fixpoint() .into_results_cursor(body); - TypeChecker { when: &self.when, body, tcx, param_env, mir_phase, storage_liveness } - .visit_body(body); + TypeChecker { + when: &self.when, + body, + tcx, + param_env, + mir_phase, + reachable_blocks: traversal::reachable_as_bitset(body), + storage_liveness, + place_cache: Vec::new(), + } + .visit_body(body); } } @@ -145,7 +158,9 @@ struct TypeChecker<'a, 'tcx> { tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, mir_phase: MirPhase, + reachable_blocks: BitSet<BasicBlock>, storage_liveness: ResultsCursor<'a, 'tcx, MaybeStorageLive>, + place_cache: Vec<PlaceRef<'tcx>>, } impl<'a, 'tcx> TypeChecker<'a, 'tcx> { @@ -219,7 +234,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { fn visit_local(&mut self, local: &Local, context: PlaceContext, location: Location) { - if context.is_use() { + if self.reachable_blocks.contains(location.block) && context.is_use() { // Uses of locals must occur while the local's storage is allocated. self.storage_liveness.seek_after_primary_effect(location); let locals_with_storage = self.storage_liveness.get(); @@ -236,13 +251,16 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { - // `Operand::Copy` is only supposed to be used with `Copy` types. - if let Operand::Copy(place) = operand { - let ty = place.ty(&self.body.local_decls, self.tcx).ty; - let span = self.body.source_info(location).span; - - if !ty.is_copy_modulo_regions(self.tcx.at(span), self.param_env) { - self.fail(location, format!("`Operand::Copy` with non-`Copy` type {}", ty)); + // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed. + if self.tcx.sess.opts.debugging_opts.validate_mir { + // `Operand::Copy` is only supposed to be used with `Copy` types. + if let Operand::Copy(place) = operand { + let ty = place.ty(&self.body.local_decls, self.tcx).ty; + let span = self.body.source_info(location).span; + + if !ty.is_copy_modulo_regions(self.tcx.at(span), self.param_env) { + self.fail(location, format!("`Operand::Copy` with non-`Copy` type {}", ty)); + } } } @@ -328,6 +346,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } _ => {} } + + self.super_statement(statement, location); } fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { @@ -346,7 +366,25 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ), ); } - for (_, target) in targets.iter() { + + let target_width = self.tcx.sess.target.pointer_width; + + let size = Size::from_bits(match switch_ty.kind() { + ty::Uint(uint) => uint.normalize(target_width).bit_width().unwrap(), + ty::Int(int) => int.normalize(target_width).bit_width().unwrap(), + ty::Char => 32, + ty::Bool => 1, + other => bug!("unhandled type: {:?}", other), + }); + + for (value, target) in targets.iter() { + if Scalar::<()>::try_from_uint(value, size).is_none() { + self.fail( + location, + format!("the value {:#x} is not a proper {:?}", value, switch_ty), + ) + } + self.check_edge(location, target, EdgeKind::Normal); } self.check_edge(location, targets.otherwise(), EdgeKind::Normal); @@ -369,7 +407,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { self.check_edge(location, *unwind, EdgeKind::Unwind); } } - TerminatorKind::Call { func, destination, cleanup, .. } => { + TerminatorKind::Call { func, args, destination, cleanup, .. } => { let func_ty = func.ty(&self.body.local_decls, self.tcx); match func_ty.kind() { ty::FnPtr(..) | ty::FnDef(..) => {} @@ -384,6 +422,32 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { if let Some(cleanup) = cleanup { self.check_edge(location, *cleanup, EdgeKind::Unwind); } + + // The call destination place and Operand::Move place used as an argument might be + // passed by a reference to the callee. Consequently they must be non-overlapping. + // Currently this simply checks for duplicate places. + self.place_cache.clear(); + if let Some((destination, _)) = destination { + self.place_cache.push(destination.as_ref()); + } + for arg in args { + if let Operand::Move(place) = arg { + self.place_cache.push(place.as_ref()); + } + } + let all_len = self.place_cache.len(); + self.place_cache.sort_unstable(); + self.place_cache.dedup(); + let has_duplicates = all_len != self.place_cache.len(); + if has_duplicates { + self.fail( + location, + format!( + "encountered overlapping memory in `Call` terminator: {:?}", + terminator.kind, + ), + ); + } } TerminatorKind::Assert { cond, target, cleanup, .. } => { let cond_ty = cond.ty(&self.body.local_decls, self.tcx); @@ -432,6 +496,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { | TerminatorKind::Unreachable | TerminatorKind::GeneratorDrop => {} } + + self.super_terminator(terminator, location); } fn visit_source_scope(&mut self, scope: &SourceScope) { diff --git a/compiler/rustc_mir/src/util/pretty.rs b/compiler/rustc_mir/src/util/pretty.rs index bd7c25bf250..5700492d6bb 100644 --- a/compiler/rustc_mir/src/util/pretty.rs +++ b/compiler/rustc_mir/src/util/pretty.rs @@ -19,6 +19,7 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt, TypeFoldable, TypeVisitor}; use rustc_target::abi::Size; +use std::ops::ControlFlow; const INDENT: &str = " "; /// Alignment for lining up comments following MIR statements @@ -548,8 +549,36 @@ fn write_scope_tree( }; for &child in children { - assert_eq!(body.source_scopes[child].parent_scope, Some(parent)); - writeln!(w, "{0:1$}scope {2} {{", "", indent, child.index())?; + let child_data = &body.source_scopes[child]; + assert_eq!(child_data.parent_scope, Some(parent)); + + let (special, span) = if let Some((callee, callsite_span)) = child_data.inlined { + ( + format!( + " (inlined {}{})", + if callee.def.requires_caller_location(tcx) { "#[track_caller] " } else { "" }, + callee + ), + Some(callsite_span), + ) + } else { + (String::new(), None) + }; + + let indented_header = format!("{0:1$}scope {2}{3} {{", "", indent, child.index(), special); + + if let Some(span) = span { + writeln!( + w, + "{0:1$} // at {2}", + indented_header, + ALIGN, + tcx.sess.source_map().span_to_string(span), + )?; + } else { + writeln!(w, "{}", indented_header)?; + } + write_scope_tree(tcx, body, scope_tree, w, child, depth + 1)?; writeln!(w, "{0:1$}}}", "", depth * INDENT.len())?; } @@ -611,7 +640,7 @@ pub fn write_allocations<'tcx>( } struct CollectAllocIds(BTreeSet<AllocId>); impl<'tcx> TypeVisitor<'tcx> for CollectAllocIds { - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { if let ty::ConstKind::Value(val) = c.val { self.0.extend(alloc_ids_from_const(val)); } diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs index aac93f313f4..cf075abc94b 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs @@ -165,7 +165,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let tcx = this.hir.tcx(); - if tcx.features().unsized_locals { + if tcx.features().unsized_fn_params { let ty = expr.ty; let span = expr.span; let param_env = this.hir.param_env; diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs index 443025c4f84..b94346fa439 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_place.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs @@ -262,10 +262,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { | ExprKind::ThreadLocalRef(_) | ExprKind::Call { .. } => { // these are not places, so we need to make a temporary. - debug_assert!(match Category::of(&expr.kind) { - Some(Category::Place) => false, - _ => true, - }); + debug_assert!(!matches!(Category::of(&expr.kind), Some(Category::Place))); let temp = unpack!(block = this.as_temp(block, expr.temp_lifetime, expr, mutability)); block.and(PlaceBuilder::from(temp)) diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs index 4033cc6cf46..2853bf887fa 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs @@ -260,10 +260,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { | ExprKind::ValueTypeAscription { .. } => { // these do not have corresponding `Rvalue` variants, // so make an operand and then return that - debug_assert!(match Category::of(&expr.kind) { - Some(Category::Rvalue(RvalueFunc::AsRvalue)) => false, - _ => true, - }); + debug_assert!(!matches!(Category::of(&expr.kind), Some(Category::Rvalue(RvalueFunc::AsRvalue)))); let operand = unpack!(block = this.as_operand(block, scope, expr)); block.and(Rvalue::Use(operand)) } diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs index a268b0b46ff..9dc596a345f 100644 --- a/compiler/rustc_mir_build/src/build/expr/into.rs +++ b/compiler/rustc_mir_build/src/build/expr/into.rs @@ -58,10 +58,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } ExprKind::NeverToAny { source } => { let source = this.hir.mirror(source); - let is_call = match source.kind { - ExprKind::Call { .. } | ExprKind::InlineAsm { .. } => true, - _ => false, - }; + let is_call = matches!(source.kind, ExprKind::Call { .. } | ExprKind::InlineAsm { .. }); // (#66975) Source could be a const of type `!`, so has to // exist in the generated MIR. diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs index b7bd67fea06..3ee15248ae2 100644 --- a/compiler/rustc_mir_build/src/build/matches/mod.rs +++ b/compiler/rustc_mir_build/src/build/matches/mod.rs @@ -231,7 +231,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let arm_end_blocks: Vec<_> = arm_candidates .into_iter() .map(|(arm, candidate)| { - debug!("lowering arm {:?}\ncanidate = {:?}", arm, candidate); + debug!("lowering arm {:?}\ncandidate = {:?}", arm, candidate); let arm_source_info = self.source_info(arm.span); let arm_scope = (arm.scope, arm_source_info); diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs index c4191900147..7bea8220ada 100644 --- a/compiler/rustc_mir_build/src/build/matches/test.rs +++ b/compiler/rustc_mir_build/src/build/matches/test.rs @@ -250,15 +250,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { place, ty, ); + } else if let [success, fail] = *make_target_blocks(self) { + assert_eq!(value.ty, ty); + let expect = self.literal_operand(test.span, value); + let val = Operand::Copy(place); + self.compare(block, success, fail, source_info, BinOp::Eq, expect, val); } else { - if let [success, fail] = *make_target_blocks(self) { - assert_eq!(value.ty, ty); - let expect = self.literal_operand(test.span, value); - let val = Operand::Copy(place); - self.compare(block, success, fail, source_info, BinOp::Eq, expect, val); - } else { - bug!("`TestKind::Eq` should have two target blocks"); - } + bug!("`TestKind::Eq` should have two target blocks"); } } diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index 899fc647493..f9995f43f5a 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -334,7 +334,7 @@ struct Builder<'a, 'tcx> { /// The vector of all scopes that we have created thus far; /// we track this for debuginfo later. - source_scopes: IndexVec<SourceScope, SourceScopeData>, + source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>, source_scope: SourceScope, /// The guard-context: each time we build the guard expression for diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs index ad6386ca12d..e91227d8357 100644 --- a/compiler/rustc_mir_build/src/build/scope.rs +++ b/compiler/rustc_mir_build/src/build/scope.rs @@ -331,13 +331,11 @@ impl DropTree { } if let DropKind::Value = drop_data.0.kind { needs_block[drop_data.1] = Block::Own; - } else { - if drop_idx != ROOT_NODE { - match &mut needs_block[drop_data.1] { - pred @ Block::None => *pred = Block::Shares(drop_idx), - pred @ Block::Shares(_) => *pred = Block::Own, - Block::Own => (), - } + } else if drop_idx != ROOT_NODE { + match &mut needs_block[drop_data.1] { + pred @ Block::None => *pred = Block::Shares(drop_idx), + pred @ Block::Shares(_) => *pred = Block::Own, + Block::Own => (), } } } @@ -461,9 +459,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let breakable_scope = self.scopes.breakable_scopes.pop().unwrap(); assert!(breakable_scope.region_scope == region_scope); let break_block = self.build_exit_tree(breakable_scope.break_drops, None); - breakable_scope.continue_drops.map(|drops| { - self.build_exit_tree(drops, loop_block); - }); + if let Some(drops) = breakable_scope.continue_drops { self.build_exit_tree(drops, loop_block); } match (normal_exit_block, break_block) { (Some(block), None) | (None, Some(block)) => block, (None, None) => self.cfg.start_new_block().unit(), @@ -705,6 +701,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.source_scopes.push(SourceScopeData { span, parent_scope: Some(parent), + inlined: None, + inlined_parent_scope: None, local_data: ClearCrossCrate::Set(scope_local_data), }) } diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs index 714041ad4e8..0866892265b 100644 --- a/compiler/rustc_mir_build/src/lib.rs +++ b/compiler/rustc_mir_build/src/lib.rs @@ -9,6 +9,7 @@ #![feature(control_flow_enum)] #![feature(crate_visibility_modifier)] #![feature(bool_to_option)] +#![feature(once_cell)] #![feature(or_patterns)] #![recursion_limit = "256"] diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs index bdef02a011b..576b537c017 100644 --- a/compiler/rustc_mir_build/src/lints.rs +++ b/compiler/rustc_mir_build/src/lints.rs @@ -1,5 +1,5 @@ use rustc_data_structures::graph::iterate::{ - ControlFlow, NodeStatus, TriColorDepthFirstSearch, TriColorVisitor, + NodeStatus, TriColorDepthFirstSearch, TriColorVisitor, }; use rustc_hir::intravisit::FnKind; use rustc_middle::hir::map::blocks::FnLikeNode; @@ -8,6 +8,7 @@ use rustc_middle::ty::subst::{GenericArg, InternalSubsts}; use rustc_middle::ty::{self, AssocItem, AssocItemContainer, Instance, TyCtxt}; use rustc_session::lint::builtin::UNCONDITIONAL_RECURSION; use rustc_span::Span; +use std::ops::ControlFlow; crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let def_id = body.source.def_id().expect_local(); @@ -71,12 +72,14 @@ impl<'mir, 'tcx> Search<'mir, 'tcx> { let func_ty = func.ty(body, tcx); if let ty::FnDef(callee, substs) = *func_ty.kind() { - let (callee, call_substs) = - if let Ok(Some(instance)) = Instance::resolve(tcx, param_env, callee, substs) { - (instance.def_id(), instance.substs) - } else { - (callee, substs) - }; + let normalized_substs = tcx.normalize_erasing_regions(param_env, substs); + let (callee, call_substs) = if let Ok(Some(instance)) = + Instance::resolve(tcx, param_env, callee, normalized_substs) + { + (instance.def_id(), instance.substs) + } else { + (callee, normalized_substs) + }; // FIXME(#57965): Make this work across function boundaries diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs index 731bd954246..6ed7ed575fc 100644 --- a/compiler/rustc_mir_build/src/thir/cx/expr.rs +++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs @@ -316,16 +316,14 @@ fn make_mirror_unadjusted<'a, 'tcx>( hir::ExprKind::Unary(hir::UnOp::UnNeg, ref arg) => { if cx.typeck_results().is_method_call(expr) { overloaded_operator(cx, expr, vec![arg.to_ref()]) - } else { - if let hir::ExprKind::Lit(ref lit) = arg.kind { - ExprKind::Literal { - literal: cx.const_eval_literal(&lit.node, expr_ty, lit.span, true), - user_ty: None, - const_id: None, - } - } else { - ExprKind::Unary { op: UnOp::Neg, arg: arg.to_ref() } + } else if let hir::ExprKind::Lit(ref lit) = arg.kind { + ExprKind::Literal { + literal: cx.const_eval_literal(&lit.node, expr_ty, lit.span, true), + user_ty: None, + const_id: None, } + } else { + ExprKind::Unary { op: UnOp::Neg, arg: arg.to_ref() } } } diff --git a/compiler/rustc_mir_build/src/thir/pattern/_match.rs b/compiler/rustc_mir_build/src/thir/pattern/_match.rs index 04de9a7a58d..e0de1351ac3 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/_match.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/_match.rs @@ -78,20 +78,26 @@ //! new pattern `p`. //! //! For example, say we have the following: +//! //! ``` -//! // x: (Option<bool>, Result<()>) -//! match x { -//! (Some(true), _) => {} -//! (None, Err(())) => {} -//! (None, Err(_)) => {} -//! } +//! // x: (Option<bool>, Result<()>) +//! match x { +//! (Some(true), _) => {} +//! (None, Err(())) => {} +//! (None, Err(_)) => {} +//! } //! ``` +//! //! Here, the matrix `P` starts as: +//! +//! ``` //! [ //! [(Some(true), _)], //! [(None, Err(()))], //! [(None, Err(_))], //! ] +//! ``` +//! //! We can tell it's not exhaustive, because `U(P, _)` is true (we're not covering //! `[(Some(false), _)]`, for instance). In addition, row 3 is not useful, because //! all the values it covers are already covered by row 2. @@ -131,8 +137,8 @@ //! S(c, (r_1, p_2, .., p_n)) //! S(c, (r_2, p_2, .., p_n)) //! -//! 2. We can pop a wildcard off the top of the stack. This is called `D(p)`, where `p` is -//! a pattern-stack. +//! 2. We can pop a wildcard off the top of the stack. This is called `S(_, p)`, where `p` is +//! a pattern-stack. Note: the paper calls this `D(p)`. //! This is used when we know there are missing constructor cases, but there might be //! existing wildcard patterns, so to check the usefulness of the matrix, we have to check //! all its *other* components. @@ -144,8 +150,8 @@ //! p_2, .., p_n //! 2.3. `p_1 = r_1 | r_2`. We expand the OR-pattern and then recurse on each resulting //! stack. -//! D((r_1, p_2, .., p_n)) -//! D((r_2, p_2, .., p_n)) +//! S(_, (r_1, p_2, .., p_n)) +//! S(_, (r_2, p_2, .., p_n)) //! //! Note that the OR-patterns are not always used directly in Rust, but are used to derive the //! exhaustive integer matching rules, so they're written here for posterity. @@ -175,13 +181,16 @@ //! we ignore all the patterns in the first column of `P` that involve other constructors. //! This is where `S(c, P)` comes in: //! `U(P, p) := U(S(c, P), S(c, p))` -//! This special case is handled in `is_useful_specialized`. //! //! For example, if `P` is: +//! +//! ``` //! [ -//! [Some(true), _], -//! [None, 0], +//! [Some(true), _], +//! [None, 0], //! ] +//! ``` +//! //! and `p` is [Some(false), 0], then we don't care about row 2 since we know `p` only //! matches values that row 2 doesn't. For row 1 however, we need to dig into the //! arguments of `Some` to know whether some new value is covered. So we compute @@ -194,14 +203,18 @@ //! before. //! That's almost correct, but only works if there were no wildcards in those first //! components. So we need to check that `p` is useful with respect to the rows that -//! start with a wildcard, if there are any. This is where `D` comes in: -//! `U(P, p) := U(D(P), D(p))` +//! start with a wildcard, if there are any. This is where `S(_, x)` comes in: +//! `U(P, p) := U(S(_, P), S(_, p))` //! //! For example, if `P` is: +//! +//! ``` //! [ //! [_, true, _], //! [None, false, 1], //! ] +//! ``` +//! //! and `p` is [_, false, _], the `Some` constructor doesn't appear in `P`. So if we //! only had row 2, we'd know that `p` is useful. However row 1 starts with a //! wildcard, so we need to check whether `U([[true, _]], [false, 1])`. @@ -215,10 +228,14 @@ //! `U(P, p) := ∃(k ϵ constructors) U(S(k, P), S(k, p))` //! //! For example, if `P` is: +//! +//! ``` //! [ //! [Some(true), _], //! [None, false], //! ] +//! ``` +//! //! and `p` is [_, false], both `None` and `Some` constructors appear in the first //! components of `P`. We will therefore try popping both constructors in turn: we //! compute `U([[true, _]], [_, false])` for the `Some` constructor, and `U([[false]], @@ -266,7 +283,7 @@ //! disjunction over every range. This is a bit more tricky to deal with: essentially we need //! to form equivalence classes of subranges of the constructor range for which the behaviour //! of the matrix `P` and new pattern `p` are the same. This is described in more -//! detail in `split_grouped_constructors`. +//! detail in `Constructor::split`. //! + If some constructors are missing from the matrix, it turns out we don't need to do //! anything special (because we know none of the integers are actually wildcards: i.e., we //! can't span wildcards using ranges). @@ -276,7 +293,8 @@ use self::Usefulness::*; use self::WitnessPreference::*; use rustc_data_structures::captures::Captures; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::sync::OnceCell; use rustc_index::vec::Idx; use super::{compare_const_vals, PatternFoldable, PatternFolder}; @@ -284,10 +302,9 @@ use super::{FieldPat, Pat, PatKind, PatRange}; use rustc_arena::TypedArena; use rustc_attr::{SignedInt, UnsignedInt}; -use rustc_errors::ErrorReported; use rustc_hir::def_id::DefId; use rustc_hir::{HirId, RangeEnd}; -use rustc_middle::mir::interpret::{truncate, AllocId, ConstValue, Pointer, Scalar}; +use rustc_middle::mir::interpret::{truncate, ConstValue}; use rustc_middle::mir::Field; use rustc_middle::ty::layout::IntegerExt; use rustc_middle::ty::{self, Const, Ty, TyCtxt}; @@ -296,110 +313,37 @@ use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::{Integer, Size, VariantIdx}; use smallvec::{smallvec, SmallVec}; -use std::borrow::Cow; use std::cmp::{self, max, min, Ordering}; -use std::convert::TryInto; use std::fmt; use std::iter::{FromIterator, IntoIterator}; use std::ops::RangeInclusive; -crate fn expand_pattern<'a, 'tcx>(cx: &MatchCheckCtxt<'a, 'tcx>, pat: Pat<'tcx>) -> Pat<'tcx> { - LiteralExpander { tcx: cx.tcx, param_env: cx.param_env }.fold_pattern(&pat) -} - -struct LiteralExpander<'tcx> { - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, +crate fn expand_pattern<'tcx>(pat: Pat<'tcx>) -> Pat<'tcx> { + LiteralExpander.fold_pattern(&pat) } -impl<'tcx> LiteralExpander<'tcx> { - /// Derefs `val` and potentially unsizes the value if `crty` is an array and `rty` a slice. - /// - /// `crty` and `rty` can differ because you can use array constants in the presence of slice - /// patterns. So the pattern may end up being a slice, but the constant is an array. We convert - /// the array to a slice in that case. - fn fold_const_value_deref( - &mut self, - val: ConstValue<'tcx>, - // the pattern's pointee type - rty: Ty<'tcx>, - // the constant's pointee type - crty: Ty<'tcx>, - ) -> ConstValue<'tcx> { - debug!("fold_const_value_deref {:?} {:?} {:?}", val, rty, crty); - match (val, &crty.kind(), &rty.kind()) { - // the easy case, deref a reference - (ConstValue::Scalar(p), x, y) if x == y => { - match p { - Scalar::Ptr(p) => { - let alloc = self.tcx.global_alloc(p.alloc_id).unwrap_memory(); - ConstValue::ByRef { alloc, offset: p.offset } - } - Scalar::Raw { .. } => { - let layout = self.tcx.layout_of(self.param_env.and(rty)).unwrap(); - if layout.is_zst() { - // Deref of a reference to a ZST is a nop. - ConstValue::Scalar(Scalar::zst()) - } else { - // FIXME(oli-obk): this is reachable for `const FOO: &&&u32 = &&&42;` - bug!("cannot deref {:#?}, {} -> {}", val, crty, rty); - } - } - } - } - // unsize array to slice if pattern is array but match value or other patterns are slice - (ConstValue::Scalar(Scalar::Ptr(p)), ty::Array(t, n), ty::Slice(u)) => { - assert_eq!(t, u); - ConstValue::Slice { - data: self.tcx.global_alloc(p.alloc_id).unwrap_memory(), - start: p.offset.bytes().try_into().unwrap(), - end: n.eval_usize(self.tcx, ty::ParamEnv::empty()).try_into().unwrap(), - } - } - // fat pointers stay the same - (ConstValue::Slice { .. }, _, _) - | (_, ty::Slice(_), ty::Slice(_)) - | (_, ty::Str, ty::Str) => val, - // FIXME(oli-obk): this is reachable for `const FOO: &&&u32 = &&&42;` being used - _ => bug!("cannot deref {:#?}, {} -> {}", val, crty, rty), - } - } -} +struct LiteralExpander; -impl<'tcx> PatternFolder<'tcx> for LiteralExpander<'tcx> { +impl<'tcx> PatternFolder<'tcx> for LiteralExpander { fn fold_pattern(&mut self, pat: &Pat<'tcx>) -> Pat<'tcx> { debug!("fold_pattern {:?} {:?} {:?}", pat, pat.ty.kind(), pat.kind); - match (pat.ty.kind(), &*pat.kind) { - (&ty::Ref(_, rty, _), &PatKind::Constant { value: Const { val, ty: const_ty } }) - if const_ty.is_ref() => - { - let crty = - if let ty::Ref(_, crty, _) = const_ty.kind() { crty } else { unreachable!() }; - if let ty::ConstKind::Value(val) = val { - Pat { - ty: pat.ty, - span: pat.span, - kind: box PatKind::Deref { - subpattern: Pat { - ty: rty, - span: pat.span, - kind: box PatKind::Constant { - value: Const::from_value( - self.tcx, - self.fold_const_value_deref(*val, rty, crty), - rty, - ), - }, - }, - }, - } - } else { - bug!("cannot deref {:#?}, {} -> {}", val, crty, rty) + match (pat.ty.kind(), pat.kind.as_ref()) { + (_, PatKind::Binding { subpattern: Some(s), .. }) => s.fold_with(self), + (_, PatKind::AscribeUserType { subpattern: s, .. }) => s.fold_with(self), + (ty::Ref(_, t, _), PatKind::Constant { .. }) if t.is_str() => { + // Treat string literal patterns as deref patterns to a `str` constant, i.e. + // `&CONST`. This expands them like other const patterns. This could have been done + // in `const_to_pat`, but that causes issues with the rest of the matching code. + let mut new_pat = pat.super_fold_with(self); + // Make a fake const pattern of type `str` (instead of `&str`). That the carried + // constant value still knows it is of type `&str`. + new_pat.ty = t; + Pat { + kind: Box::new(PatKind::Deref { subpattern: new_pat }), + span: pat.span, + ty: pat.ty, } } - - (_, &PatKind::Binding { subpattern: Some(ref s), .. }) => s.fold_with(self), - (_, &PatKind::AscribeUserType { subpattern: ref s, .. }) => s.fold_with(self), _ => pat.super_fold_with(self), } } @@ -407,49 +351,46 @@ impl<'tcx> PatternFolder<'tcx> for LiteralExpander<'tcx> { impl<'tcx> Pat<'tcx> { pub(super) fn is_wildcard(&self) -> bool { - match *self.kind { - PatKind::Binding { subpattern: None, .. } | PatKind::Wild => true, - _ => false, - } + matches!(*self.kind, PatKind::Binding { subpattern: None, .. } | PatKind::Wild) } } /// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]` /// works well. -#[derive(Debug, Clone, PartialEq)] -crate struct PatStack<'p, 'tcx>(SmallVec<[&'p Pat<'tcx>; 2]>); +#[derive(Debug, Clone)] +crate struct PatStack<'p, 'tcx> { + pats: SmallVec<[&'p Pat<'tcx>; 2]>, + /// Cache for the constructor of the head + head_ctor: OnceCell<Constructor<'tcx>>, +} impl<'p, 'tcx> PatStack<'p, 'tcx> { crate fn from_pattern(pat: &'p Pat<'tcx>) -> Self { - PatStack(smallvec![pat]) + Self::from_vec(smallvec![pat]) } fn from_vec(vec: SmallVec<[&'p Pat<'tcx>; 2]>) -> Self { - PatStack(vec) - } - - fn from_slice(s: &[&'p Pat<'tcx>]) -> Self { - PatStack(SmallVec::from_slice(s)) + PatStack { pats: vec, head_ctor: OnceCell::new() } } fn is_empty(&self) -> bool { - self.0.is_empty() + self.pats.is_empty() } fn len(&self) -> usize { - self.0.len() + self.pats.len() } fn head(&self) -> &'p Pat<'tcx> { - self.0[0] + self.pats[0] } - fn to_tail(&self) -> Self { - PatStack::from_slice(&self.0[1..]) + fn head_ctor<'a>(&'a self, cx: &MatchCheckCtxt<'p, 'tcx>) -> &'a Constructor<'tcx> { + self.head_ctor.get_or_init(|| pat_constructor(cx, self.head())) } fn iter(&self) -> impl Iterator<Item = &Pat<'tcx>> { - self.0.iter().copied() + self.pats.iter().copied() } // If the first pattern is an or-pattern, expand this pattern. Otherwise, return `None`. @@ -461,7 +402,7 @@ impl<'p, 'tcx> PatStack<'p, 'tcx> { pats.iter() .map(|pat| { let mut new_patstack = PatStack::from_pattern(pat); - new_patstack.0.extend_from_slice(&self.0[1..]); + new_patstack.pats.extend_from_slice(&self.pats[1..]); new_patstack }) .collect(), @@ -471,27 +412,29 @@ impl<'p, 'tcx> PatStack<'p, 'tcx> { } } - /// This computes `D(self)`. See top of the file for explanations. - fn specialize_wildcard(&self) -> Option<Self> { - if self.head().is_wildcard() { Some(self.to_tail()) } else { None } - } - - /// This computes `S(constructor, self)`. See top of the file for explanations. - fn specialize_constructor( - &self, - cx: &mut MatchCheckCtxt<'p, 'tcx>, - constructor: &Constructor<'tcx>, - ctor_wild_subpatterns: &Fields<'p, 'tcx>, - ) -> Option<PatStack<'p, 'tcx>> { - let new_fields = - specialize_one_pattern(cx, self.head(), constructor, ctor_wild_subpatterns)?; - Some(new_fields.push_on_patstack(&self.0[1..])) + /// This computes `S(self.head_ctor(), self)`. See top of the file for explanations. + /// + /// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing + /// fields filled with wild patterns. + /// + /// This is roughly the inverse of `Constructor::apply`. + fn pop_head_constructor(&self, ctor_wild_subpatterns: &Fields<'p, 'tcx>) -> PatStack<'p, 'tcx> { + // We pop the head pattern and push the new fields extracted from the arguments of + // `self.head()`. + let new_fields = ctor_wild_subpatterns.replace_with_pattern_arguments(self.head()); + new_fields.push_on_patstack(&self.pats[1..]) } } impl<'p, 'tcx> Default for PatStack<'p, 'tcx> { fn default() -> Self { - PatStack(smallvec![]) + Self::from_vec(smallvec![]) + } +} + +impl<'p, 'tcx> PartialEq for PatStack<'p, 'tcx> { + fn eq(&self, other: &Self) -> bool { + self.pats == other.pats } } @@ -500,40 +443,19 @@ impl<'p, 'tcx> FromIterator<&'p Pat<'tcx>> for PatStack<'p, 'tcx> { where T: IntoIterator<Item = &'p Pat<'tcx>>, { - PatStack(iter.into_iter().collect()) + Self::from_vec(iter.into_iter().collect()) } } -/// Depending on the match patterns, the specialization process might be able to use a fast path. -/// Tracks whether we can use the fast path and the lookup table needed in those cases. -#[derive(Clone, Debug, PartialEq)] -enum SpecializationCache { - /// Patterns consist of only enum variants. - /// Variant patterns does not intersect with each other (in contrast to range patterns), - /// so it is possible to precompute the result of `Matrix::specialize_constructor` at a - /// lower computational complexity. - /// `lookup` is responsible for holding the precomputed result of - /// `Matrix::specialize_constructor`, while `wilds` is used for two purposes: the first one is - /// the precomputed result of `Matrix::specialize_wildcard`, and the second is to be used as a - /// fallback for `Matrix::specialize_constructor` when it tries to apply a constructor that - /// has not been seen in the `Matrix`. See `update_cache` for further explanations. - Variants { lookup: FxHashMap<DefId, SmallVec<[usize; 1]>>, wilds: SmallVec<[usize; 1]> }, - /// Does not belong to the cases above, use the slow path. - Incompatible, -} - /// A 2D matrix. #[derive(Clone, PartialEq)] crate struct Matrix<'p, 'tcx> { patterns: Vec<PatStack<'p, 'tcx>>, - cache: SpecializationCache, } impl<'p, 'tcx> Matrix<'p, 'tcx> { crate fn empty() -> Self { - // Use `SpecializationCache::Incompatible` as a placeholder; we will initialize it on the - // first call to `push`. See the first half of `update_cache`. - Matrix { patterns: vec![], cache: SpecializationCache::Incompatible } + Matrix { patterns: vec![] } } /// Pushes a new row to the matrix. If the row starts with an or-pattern, this expands it. @@ -546,70 +468,6 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> { } } else { self.patterns.push(row); - self.update_cache(self.patterns.len() - 1); - } - } - - fn update_cache(&mut self, idx: usize) { - let row = &self.patterns[idx]; - // We don't know which kind of cache could be used until we see the first row; therefore an - // empty `Matrix` is initialized with `SpecializationCache::Empty`, then the cache is - // assigned the appropriate variant below on the first call to `push`. - if self.patterns.is_empty() { - self.cache = if row.is_empty() { - SpecializationCache::Incompatible - } else { - match *row.head().kind { - PatKind::Variant { .. } => SpecializationCache::Variants { - lookup: FxHashMap::default(), - wilds: SmallVec::new(), - }, - // Note: If the first pattern is a wildcard, then all patterns after that is not - // useful. The check is simple enough so we treat it as the same as unsupported - // patterns. - _ => SpecializationCache::Incompatible, - } - }; - } - // Update the cache. - match &mut self.cache { - SpecializationCache::Variants { ref mut lookup, ref mut wilds } => { - let head = row.head(); - match *head.kind { - _ if head.is_wildcard() => { - // Per rule 1.3 in the top-level comments, a wildcard pattern is included in - // the result of `specialize_constructor` for *any* `Constructor`. - // We push the wildcard pattern to the precomputed result for constructors - // that we have seen before; results for constructors we have not yet seen - // defaults to `wilds`, which is updated right below. - for (_, v) in lookup.iter_mut() { - v.push(idx); - } - // Per rule 2.1 and 2.2 in the top-level comments, only wildcard patterns - // are included in the result of `specialize_wildcard`. - // What we do here is to track the wildcards we have seen; so in addition to - // acting as the precomputed result of `specialize_wildcard`, `wilds` also - // serves as the default value of `specialize_constructor` for constructors - // that are not in `lookup`. - wilds.push(idx); - } - PatKind::Variant { adt_def, variant_index, .. } => { - // Handle the cases of rule 1.1 and 1.2 in the top-level comments. - // A variant pattern can only be included in the results of - // `specialize_constructor` for a particular constructor, therefore we are - // using a HashMap to track that. - lookup - .entry(adt_def.variants[variant_index].def_id) - // Default to `wilds` for absent keys. See above for an explanation. - .or_insert_with(|| wilds.clone()) - .push(idx); - } - _ => { - self.cache = SpecializationCache::Incompatible; - } - } - } - SpecializationCache::Incompatible => {} } } @@ -618,78 +476,26 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> { self.patterns.iter().map(|r| r.head()) } - /// This computes `D(self)`. See top of the file for explanations. - fn specialize_wildcard(&self) -> Self { - match &self.cache { - SpecializationCache::Variants { wilds, .. } => { - let result = - wilds.iter().filter_map(|&i| self.patterns[i].specialize_wildcard()).collect(); - // When debug assertions are enabled, check the results against the "slow path" - // result. - debug_assert_eq!( - result, - Self { - patterns: self.patterns.clone(), - cache: SpecializationCache::Incompatible - } - .specialize_wildcard() - ); - result - } - SpecializationCache::Incompatible => { - self.patterns.iter().filter_map(|r| r.specialize_wildcard()).collect() - } - } + /// Iterate over the first constructor of each row + fn head_ctors<'a>( + &'a self, + cx: &'a MatchCheckCtxt<'p, 'tcx>, + ) -> impl Iterator<Item = &'a Constructor<'tcx>> + Captures<'a> + Captures<'p> { + self.patterns.iter().map(move |r| r.head_ctor(cx)) } /// This computes `S(constructor, self)`. See top of the file for explanations. fn specialize_constructor( &self, - cx: &mut MatchCheckCtxt<'p, 'tcx>, - constructor: &Constructor<'tcx>, + pcx: PatCtxt<'_, 'p, 'tcx>, + ctor: &Constructor<'tcx>, ctor_wild_subpatterns: &Fields<'p, 'tcx>, ) -> Matrix<'p, 'tcx> { - match &self.cache { - SpecializationCache::Variants { lookup, wilds } => { - let result: Self = if let Constructor::Variant(id) = constructor { - lookup - .get(id) - // Default to `wilds` for absent keys. See `update_cache` for an explanation. - .unwrap_or(&wilds) - .iter() - .filter_map(|&i| { - self.patterns[i].specialize_constructor( - cx, - constructor, - ctor_wild_subpatterns, - ) - }) - .collect() - } else { - unreachable!() - }; - // When debug assertions are enabled, check the results against the "slow path" - // result. - debug_assert_eq!( - result, - Matrix { - patterns: self.patterns.clone(), - cache: SpecializationCache::Incompatible - } - .specialize_constructor( - cx, - constructor, - ctor_wild_subpatterns - ) - ); - result - } - SpecializationCache::Incompatible => self - .patterns - .iter() - .filter_map(|r| r.specialize_constructor(cx, constructor, ctor_wild_subpatterns)) - .collect(), - } + self.patterns + .iter() + .filter(|r| ctor.is_covered_by(pcx, r.head_ctor(pcx.cx))) + .map(|r| r.pop_head_constructor(ctor_wild_subpatterns)) + .collect() } } @@ -707,6 +513,7 @@ impl<'p, 'tcx> Matrix<'p, 'tcx> { /// +++++++++++++++++++++++++++++ /// + _ + [_, _, tail @ ..] + /// +++++++++++++++++++++++++++++ +/// ``` impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "\n")?; @@ -812,46 +619,6 @@ impl SliceKind { VarLen(prefix, suffix) => prefix + suffix <= other_len, } } - - /// Returns a collection of slices that spans the values covered by `self`, subtracted by the - /// values covered by `other`: i.e., `self \ other` (in set notation). - fn subtract(self, other: Self) -> SmallVec<[Self; 1]> { - // Remember, `VarLen(i, j)` covers the union of `FixedLen` from `i + j` to infinity. - // Naming: we remove the "neg" constructors from the "pos" ones. - match self { - FixedLen(pos_len) => { - if other.covers_length(pos_len) { - smallvec![] - } else { - smallvec![self] - } - } - VarLen(pos_prefix, pos_suffix) => { - let pos_len = pos_prefix + pos_suffix; - match other { - FixedLen(neg_len) => { - if neg_len < pos_len { - smallvec![self] - } else { - (pos_len..neg_len) - .map(FixedLen) - // We know that `neg_len + 1 >= pos_len >= pos_suffix`. - .chain(Some(VarLen(neg_len + 1 - pos_suffix, pos_suffix))) - .collect() - } - } - VarLen(neg_prefix, neg_suffix) => { - let neg_len = neg_prefix + neg_suffix; - if neg_len <= pos_len { - smallvec![] - } else { - (pos_len..neg_len).map(FixedLen).collect() - } - } - } - } - } - } } /// A constructor for array and slice patterns. @@ -864,33 +631,142 @@ struct Slice { } impl Slice { - /// Returns what patterns this constructor covers: either fixed-length patterns or - /// variable-length patterns. - fn pattern_kind(self) -> SliceKind { - match self { - Slice { array_len: Some(len), kind: VarLen(prefix, suffix) } - if prefix + suffix == len => - { - FixedLen(len) - } - _ => self.kind, - } + fn new(array_len: Option<u64>, kind: SliceKind) -> Self { + let kind = match (array_len, kind) { + // If the middle `..` is empty, we effectively have a fixed-length pattern. + (Some(len), VarLen(prefix, suffix)) if prefix + suffix >= len => FixedLen(len), + _ => kind, + }; + Slice { array_len, kind } + } + + fn arity(self) -> u64 { + self.kind.arity() } - /// Returns what values this constructor covers: either values of only one given length, or - /// values of length above a given length. - /// This is different from `pattern_kind()` because in some cases the pattern only takes into - /// account a subset of the entries of the array, but still only captures values of a given + /// The exhaustiveness-checking paper does not include any details on + /// checking variable-length slice patterns. However, they may be + /// matched by an infinite collection of fixed-length array patterns. + /// + /// Checking the infinite set directly would take an infinite amount + /// of time. However, it turns out that for each finite set of + /// patterns `P`, all sufficiently large array lengths are equivalent: + /// + /// Each slice `s` with a "sufficiently-large" length `l ≥ L` that applies + /// to exactly the subset `Pâ‚œ` of `P` can be transformed to a slice + /// `sₘ` for each sufficiently-large length `m` that applies to exactly + /// the same subset of `P`. + /// + /// Because of that, each witness for reachability-checking of one + /// of the sufficiently-large lengths can be transformed to an + /// equally-valid witness of any other length, so we only have + /// to check slices of the "minimal sufficiently-large length" + /// and less. + /// + /// Note that the fact that there is a *single* `sₘ` for each `m` + /// not depending on the specific pattern in `P` is important: if + /// you look at the pair of patterns + /// `[true, ..]` + /// `[.., false]` + /// Then any slice of length ≥1 that matches one of these two + /// patterns can be trivially turned to a slice of any + /// other length ≥1 that matches them and vice-versa, + /// but the slice of length 2 `[false, true]` that matches neither + /// of these patterns can't be turned to a slice from length 1 that + /// matches neither of these patterns, so we have to consider + /// slices from length 2 there. + /// + /// Now, to see that that length exists and find it, observe that slice + /// patterns are either "fixed-length" patterns (`[_, _, _]`) or + /// "variable-length" patterns (`[_, .., _]`). + /// + /// For fixed-length patterns, all slices with lengths *longer* than + /// the pattern's length have the same outcome (of not matching), so + /// as long as `L` is greater than the pattern's length we can pick + /// any `sₘ` from that length and get the same result. + /// + /// For variable-length patterns, the situation is more complicated, + /// because as seen above the precise value of `sₘ` matters. + /// + /// However, for each variable-length pattern `p` with a prefix of length + /// `plâ‚š` and suffix of length `slâ‚š`, only the first `plâ‚š` and the last + /// `slâ‚š` elements are examined. + /// + /// Therefore, as long as `L` is positive (to avoid concerns about empty + /// types), all elements after the maximum prefix length and before + /// the maximum suffix length are not examined by any variable-length + /// pattern, and therefore can be added/removed without affecting + /// them - creating equivalent patterns from any sufficiently-large /// length. - fn value_kind(self) -> SliceKind { - match self { - Slice { array_len: Some(len), kind: VarLen(_, _) } => FixedLen(len), - _ => self.kind, + /// + /// Of course, if fixed-length patterns exist, we must be sure + /// that our length is large enough to miss them all, so + /// we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))` + /// + /// for example, with the above pair of patterns, all elements + /// but the first and last can be added/removed, so any + /// witness of length ≥2 (say, `[false, false, true]`) can be + /// turned to a witness from any other length ≥2. + fn split<'p, 'tcx>(self, pcx: PatCtxt<'_, 'p, 'tcx>) -> SmallVec<[Constructor<'tcx>; 1]> { + let (self_prefix, self_suffix) = match self.kind { + VarLen(self_prefix, self_suffix) => (self_prefix, self_suffix), + _ => return smallvec![Slice(self)], + }; + + let head_ctors = pcx.matrix.head_ctors(pcx.cx).filter(|c| !c.is_wildcard()); + + let mut max_prefix_len = self_prefix; + let mut max_suffix_len = self_suffix; + let mut max_fixed_len = 0; + + for ctor in head_ctors { + if let Slice(slice) = ctor { + match slice.kind { + FixedLen(len) => { + max_fixed_len = cmp::max(max_fixed_len, len); + } + VarLen(prefix, suffix) => { + max_prefix_len = cmp::max(max_prefix_len, prefix); + max_suffix_len = cmp::max(max_suffix_len, suffix); + } + } + } else { + bug!("unexpected ctor for slice type: {:?}", ctor); + } + } + + // For diagnostics, we keep the prefix and suffix lengths separate, so in the case + // where `max_fixed_len + 1` is the largest, we adapt `max_prefix_len` accordingly, + // so that `L = max_prefix_len + max_suffix_len`. + if max_fixed_len + 1 >= max_prefix_len + max_suffix_len { + // The subtraction can't overflow thanks to the above check. + // The new `max_prefix_len` is also guaranteed to be larger than its previous + // value. + max_prefix_len = max_fixed_len + 1 - max_suffix_len; + } + + let final_slice = VarLen(max_prefix_len, max_suffix_len); + let final_slice = Slice::new(self.array_len, final_slice); + match self.array_len { + Some(_) => smallvec![Slice(final_slice)], + None => { + // `self` originally covered the range `(self.arity()..infinity)`. We split that + // range into two: lengths smaller than `final_slice.arity()` are treated + // independently as fixed-lengths slices, and lengths above are captured by + // `final_slice`. + let smaller_lengths = (self.arity()..final_slice.arity()).map(FixedLen); + smaller_lengths + .map(|kind| Slice::new(self.array_len, kind)) + .chain(Some(final_slice)) + .map(Slice) + .collect() + } } } - fn arity(self) -> u64 { - self.pattern_kind().arity() + /// See `Constructor::is_covered_by` + fn is_covered_by(self, other: Self) -> bool { + other.kind.covers_length(self.arity()) } } @@ -898,7 +774,7 @@ impl Slice { /// the constructor. See also `Fields`. /// /// `pat_constructor` retrieves the constructor corresponding to a pattern. -/// `specialize_one_pattern` returns the list of fields corresponding to a pattern, given a +/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a /// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and /// `Fields`. #[derive(Clone, Debug, PartialEq)] @@ -908,135 +784,204 @@ enum Constructor<'tcx> { Single, /// Enum variants. Variant(DefId), - /// Literal values. - ConstantValue(&'tcx ty::Const<'tcx>), /// Ranges of integer literal values (`2`, `2..=5` or `2..5`). IntRange(IntRange<'tcx>), /// Ranges of floating-point literal values (`2.0..=5.2`). FloatRange(&'tcx ty::Const<'tcx>, &'tcx ty::Const<'tcx>, RangeEnd), + /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately. + Str(&'tcx ty::Const<'tcx>), /// Array and slice patterns. Slice(Slice), - /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. + /// Constants that must not be matched structurally. They are treated as black + /// boxes for the purposes of exhaustiveness: we must not inspect them, and they + /// don't count towards making a match exhaustive. + Opaque, + /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used + /// for those types for which we cannot list constructors explicitly, like `f64` and `str`. NonExhaustive, + /// Wildcard pattern. + Wildcard, } impl<'tcx> Constructor<'tcx> { - fn is_slice(&self) -> bool { + fn is_wildcard(&self) -> bool { + matches!(self, Wildcard) + } + + fn as_int_range(&self) -> Option<&IntRange<'tcx>> { match self { - Slice(_) => true, - _ => false, + IntRange(range) => Some(range), + _ => None, } } - fn variant_index_for_adt<'a>( - &self, - cx: &MatchCheckCtxt<'a, 'tcx>, - adt: &'tcx ty::AdtDef, - ) -> VariantIdx { + fn as_slice(&self) -> Option<Slice> { + match self { + Slice(slice) => Some(*slice), + _ => None, + } + } + + fn variant_index_for_adt(&self, adt: &'tcx ty::AdtDef) -> VariantIdx { match *self { Variant(id) => adt.variant_index_with_id(id), Single => { assert!(!adt.is_enum()); VariantIdx::new(0) } - ConstantValue(c) => cx - .tcx - .destructure_const(cx.param_env.and(c)) - .variant - .expect("destructed const of adt without variant id"), _ => bug!("bad constructor {:?} for adt {:?}", self, adt), } } - // Returns the set of constructors covered by `self` but not by - // anything in `other_ctors`. - fn subtract_ctors(&self, other_ctors: &Vec<Constructor<'tcx>>) -> Vec<Constructor<'tcx>> { - if other_ctors.is_empty() { - return vec![self.clone()]; - } + /// Some constructors (namely `Wildcard`, `IntRange` and `Slice`) actually stand for a set of actual + /// constructors (like variants, integers or fixed-sized slices). When specializing for these + /// constructors, we want to be specialising for the actual underlying constructors. + /// Naively, we would simply return the list of constructors they correspond to. We instead are + /// more clever: if there are constructors that we know will behave the same wrt the current + /// matrix, we keep them grouped. For example, all slices of a sufficiently large length + /// will either be all useful or all non-useful with a given matrix. + /// + /// See the branches for details on how the splitting is done. + /// + /// This function may discard some irrelevant constructors if this preserves behavior and + /// diagnostics. Eg. for the `_` case, we ignore the constructors already present in the + /// matrix, unless all of them are. + /// + /// `hir_id` is `None` when we're evaluating the wildcard pattern. In that case we do not want + /// to lint for overlapping ranges. + fn split<'p>(&self, pcx: PatCtxt<'_, 'p, 'tcx>, hir_id: Option<HirId>) -> SmallVec<[Self; 1]> { + debug!("Constructor::split({:#?}, {:#?})", self, pcx.matrix); match self { - // Those constructors can only match themselves. - Single | Variant(_) | ConstantValue(..) | FloatRange(..) => { - if other_ctors.iter().any(|c| c == self) { vec![] } else { vec![self.clone()] } + Wildcard => Constructor::split_wildcard(pcx), + // Fast-track if the range is trivial. In particular, we don't do the overlapping + // ranges check. + IntRange(ctor_range) + if ctor_range.treat_exhaustively(pcx.cx.tcx) && !ctor_range.is_singleton() => + { + ctor_range.split(pcx, hir_id) } - &Slice(slice) => { - let mut other_slices = other_ctors - .iter() - .filter_map(|c: &Constructor<'_>| match c { - Slice(slice) => Some(*slice), - // FIXME(oli-obk): implement `deref` for `ConstValue` - ConstantValue(..) => None, - _ => bug!("bad slice pattern constructor {:?}", c), - }) - .map(Slice::value_kind); + Slice(slice @ Slice { kind: VarLen(..), .. }) => slice.split(pcx), + // Any other constructor can be used unchanged. + _ => smallvec![self.clone()], + } + } - match slice.value_kind() { - FixedLen(self_len) => { - if other_slices.any(|other_slice| other_slice.covers_length(self_len)) { - vec![] - } else { - vec![Slice(slice)] - } - } - kind @ VarLen(..) => { - let mut remaining_slices = vec![kind]; - - // For each used slice, subtract from the current set of slices. - for other_slice in other_slices { - remaining_slices = remaining_slices - .into_iter() - .flat_map(|remaining_slice| remaining_slice.subtract(other_slice)) - .collect(); + /// For wildcards, there are two groups of constructors: there are the constructors actually + /// present in the matrix (`head_ctors`), and the constructors not present (`missing_ctors`). + /// Two constructors that are not in the matrix will either both be caught (by a wildcard), or + /// both not be caught. Therefore we can keep the missing constructors grouped together. + fn split_wildcard<'p>(pcx: PatCtxt<'_, 'p, 'tcx>) -> SmallVec<[Self; 1]> { + // Missing constructors are those that are not matched by any non-wildcard patterns in the + // current column. We only fully construct them on-demand, because they're rarely used and + // can be big. + let missing_ctors = MissingConstructors::new(pcx); + if missing_ctors.is_empty(pcx) { + // All the constructors are present in the matrix, so we just go through them all. + // We must also split them first. + missing_ctors.all_ctors + } else { + // Some constructors are missing, thus we can specialize with the wildcard constructor, + // which will stand for those constructors that are missing, and behaves like any of + // them. + smallvec![Wildcard] + } + } - // If the constructors that have been considered so far already cover - // the entire range of `self`, no need to look at more constructors. - if remaining_slices.is_empty() { - break; - } - } + /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`. + /// For the simple cases, this is simply checking for equality. For the "grouped" constructors, + /// this checks for inclusion. + fn is_covered_by<'p>(&self, pcx: PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool { + // This must be kept in sync with `is_covered_by_any`. + match (self, other) { + // Wildcards cover anything + (_, Wildcard) => true, + // Wildcards are only covered by wildcards + (Wildcard, _) => false, - remaining_slices - .into_iter() - .map(|kind| Slice { array_len: slice.array_len, kind }) - .map(Slice) - .collect() + (Single, Single) => true, + (Variant(self_id), Variant(other_id)) => self_id == other_id, + + (IntRange(self_range), IntRange(other_range)) => { + self_range.is_covered_by(pcx, other_range) + } + ( + FloatRange(self_from, self_to, self_end), + FloatRange(other_from, other_to, other_end), + ) => { + match ( + compare_const_vals(pcx.cx.tcx, self_to, other_to, pcx.cx.param_env, pcx.ty), + compare_const_vals(pcx.cx.tcx, self_from, other_from, pcx.cx.param_env, pcx.ty), + ) { + (Some(to), Some(from)) => { + (from == Ordering::Greater || from == Ordering::Equal) + && (to == Ordering::Less + || (other_end == self_end && to == Ordering::Equal)) } + _ => false, } } - IntRange(self_range) => { - let mut remaining_ranges = vec![self_range.clone()]; - for other_ctor in other_ctors { - if let IntRange(other_range) = other_ctor { - if other_range == self_range { - // If the `self` range appears directly in a `match` arm, we can - // eliminate it straight away. - remaining_ranges = vec![]; - } else { - // Otherwise explicitly compute the remaining ranges. - remaining_ranges = other_range.subtract_from(remaining_ranges); - } - - // If the ranges that have been considered so far already cover the entire - // range of values, we can return early. - if remaining_ranges.is_empty() { - break; - } - } + (Str(self_val), Str(other_val)) => { + // FIXME: there's probably a more direct way of comparing for equality + match compare_const_vals(pcx.cx.tcx, self_val, other_val, pcx.cx.param_env, pcx.ty) + { + Some(comparison) => comparison == Ordering::Equal, + None => false, } - - // Convert the ranges back into constructors. - remaining_ranges.into_iter().map(IntRange).collect() } + (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice), + + // We are trying to inspect an opaque constant. Thus we skip the row. + (Opaque, _) | (_, Opaque) => false, + // Only a wildcard pattern can match the special extra constructor. + (NonExhaustive, _) => false, + + _ => span_bug!( + pcx.span, + "trying to compare incompatible constructors {:?} and {:?}", + self, + other + ), + } + } + + /// Faster version of `is_covered_by` when applied to many constructors. `used_ctors` is + /// assumed to be built from `matrix.head_ctors()` with wildcards filtered out, and `self` is + /// assumed to have been split from a wildcard. + fn is_covered_by_any<'p>( + &self, + pcx: PatCtxt<'_, 'p, 'tcx>, + used_ctors: &[Constructor<'tcx>], + ) -> bool { + if used_ctors.is_empty() { + return false; + } + + // This must be kept in sync with `is_covered_by`. + match self { + // If `self` is `Single`, `used_ctors` cannot contain anything else than `Single`s. + Single => !used_ctors.is_empty(), + Variant(_) => used_ctors.iter().any(|c| c == self), + IntRange(range) => used_ctors + .iter() + .filter_map(|c| c.as_int_range()) + .any(|other| range.is_covered_by(pcx, other)), + Slice(slice) => used_ctors + .iter() + .filter_map(|c| c.as_slice()) + .any(|other| slice.is_covered_by(other)), // This constructor is never covered by anything else - NonExhaustive => vec![NonExhaustive], + NonExhaustive => false, + Str(..) | FloatRange(..) | Opaque | Wildcard => { + bug!("found unexpected ctor in all_ctors: {:?}", self) + } } } /// Apply a constructor to a list of patterns, yielding a new pattern. `pats` /// must have as many elements as this constructor's arity. /// - /// This is roughly the inverse of `specialize_one_pattern`. + /// This is roughly the inverse of `specialize_constructor`. /// /// Examples: /// `self`: `Constructor::Single` @@ -1048,28 +993,23 @@ impl<'tcx> Constructor<'tcx> { /// `ty`: `Option<bool>` /// `pats`: `[false]` /// returns `Some(false)` - fn apply<'p>( - &self, - cx: &MatchCheckCtxt<'p, 'tcx>, - ty: Ty<'tcx>, - fields: Fields<'p, 'tcx>, - ) -> Pat<'tcx> { + fn apply<'p>(&self, pcx: PatCtxt<'_, 'p, 'tcx>, fields: Fields<'p, 'tcx>) -> Pat<'tcx> { let mut subpatterns = fields.all_patterns(); let pat = match self { - Single | Variant(_) => match ty.kind() { + Single | Variant(_) => match pcx.ty.kind() { ty::Adt(..) | ty::Tuple(..) => { let subpatterns = subpatterns .enumerate() .map(|(i, p)| FieldPat { field: Field::new(i), pattern: p }) .collect(); - if let ty::Adt(adt, substs) = ty.kind() { + if let ty::Adt(adt, substs) = pcx.ty.kind() { if adt.is_enum() { PatKind::Variant { adt_def: adt, substs, - variant_index: self.variant_index_for_adt(cx, adt), + variant_index: self.variant_index_for_adt(adt), subpatterns, } } else { @@ -1079,11 +1019,15 @@ impl<'tcx> Constructor<'tcx> { PatKind::Leaf { subpatterns } } } + // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should + // be careful to reconstruct the correct constant pattern here. However a string + // literal pattern will never be reported as a non-exhaustiveness witness, so we + // can ignore this issue. ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() }, - ty::Slice(_) | ty::Array(..) => bug!("bad slice pattern {:?} {:?}", self, ty), + ty::Slice(_) | ty::Array(..) => bug!("bad slice pattern {:?} {:?}", self, pcx.ty), _ => PatKind::Wild, }, - Slice(slice) => match slice.pattern_kind() { + Slice(slice) => match slice.kind { FixedLen(_) => { PatKind::Slice { prefix: subpatterns.collect(), slice: None, suffix: vec![] } } @@ -1104,22 +1048,21 @@ impl<'tcx> Constructor<'tcx> { } else { subpatterns.collect() }; - let wild = Pat::wildcard_from_ty(ty); + let wild = Pat::wildcard_from_ty(pcx.ty); PatKind::Slice { prefix, slice: Some(wild), suffix } } }, - &ConstantValue(value) => PatKind::Constant { value }, + &Str(value) => PatKind::Constant { value }, &FloatRange(lo, hi, end) => PatKind::Range(PatRange { lo, hi, end }), - IntRange(range) => return range.to_pat(cx.tcx), + IntRange(range) => return range.to_pat(pcx.cx.tcx), NonExhaustive => PatKind::Wild, + Opaque => bug!("we should not try to apply an opaque constructor"), + Wildcard => bug!( + "trying to apply a wildcard constructor; this should have been done in `apply_constructors`" + ), }; - Pat { ty, span: DUMMY_SP, kind: Box::new(pat) } - } - - /// Like `apply`, but where all the subpatterns are wildcards `_`. - fn apply_wildcards<'a>(&self, cx: &MatchCheckCtxt<'a, 'tcx>, ty: Ty<'tcx>) -> Pat<'tcx> { - self.apply(cx, ty, Fields::wildcards(cx, self, ty)) + Pat { ty: pcx.ty, span: DUMMY_SP, kind: Box::new(pat) } } } @@ -1186,12 +1129,6 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { Fields::Slice(std::slice::from_ref(pat)) } - /// Construct a new `Fields` from the given patterns. You must be sure those patterns can't - /// contain fields that need to be filtered out. When in doubt, prefer `replace_fields`. - fn from_slice_unfiltered(pats: &'p [Pat<'tcx>]) -> Self { - Fields::Slice(pats) - } - /// Convenience; internal use. fn wildcards_from_tys( cx: &MatchCheckCtxt<'p, 'tcx>, @@ -1203,11 +1140,9 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { } /// Creates a new list of wildcard fields for a given constructor. - fn wildcards( - cx: &MatchCheckCtxt<'p, 'tcx>, - constructor: &Constructor<'tcx>, - ty: Ty<'tcx>, - ) -> Self { + fn wildcards(pcx: PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self { + let ty = pcx.ty; + let cx = pcx.cx; let wildcard_from_ty = |ty| &*cx.pattern_arena.alloc(Pat::wildcard_from_ty(ty)); let ret = match constructor { @@ -1221,7 +1156,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { // Use T as the sub pattern type of Box<T>. Fields::from_single_pattern(wildcard_from_ty(substs.type_at(0))) } else { - let variant = &adt.variants[constructor.variant_index_for_adt(cx, adt)]; + let variant = &adt.variants[constructor.variant_index_for_adt(adt)]; // Whether we must not match the fields of this variant exhaustively. let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did.is_local(); @@ -1260,7 +1195,7 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { } } } - _ => Fields::empty(), + _ => bug!("Unexpected type for `Single` constructor: {:?}", ty), }, Slice(slice) => match *ty.kind() { ty::Slice(ty) | ty::Array(ty, _) => { @@ -1269,7 +1204,9 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { } _ => bug!("bad slice pattern {:?} {:?}", constructor, ty), }, - ConstantValue(..) | FloatRange(..) | IntRange(..) | NonExhaustive => Fields::empty(), + Str(..) | FloatRange(..) | IntRange(..) | NonExhaustive | Opaque | Wildcard => { + Fields::empty() + } }; debug!("Fields::wildcards({:?}, {:?}) = {:#?}", constructor, ty, ret); ret @@ -1367,6 +1304,45 @@ impl<'p, 'tcx> Fields<'p, 'tcx> { } } + /// Replaces contained fields with the arguments of the given pattern. Only use on a pattern + /// that is compatible with the constructor used to build `self`. + /// This is meant to be used on the result of `Fields::wildcards()`. The idea is that + /// `wildcards` constructs a list of fields where all entries are wildcards, and the pattern + /// provided to this function fills some of the fields with non-wildcards. + /// In the following example `Fields::wildcards` would return `[_, _, _, _]`. If we call + /// `replace_with_pattern_arguments` on it with the pattern, the result will be `[Some(0), _, + /// _, _]`. + /// ```rust + /// let x: [Option<u8>; 4] = foo(); + /// match x { + /// [Some(0), ..] => {} + /// } + /// ``` + /// This is guaranteed to preserve the number of patterns in `self`. + fn replace_with_pattern_arguments(&self, pat: &'p Pat<'tcx>) -> Self { + match pat.kind.as_ref() { + PatKind::Deref { subpattern } => { + assert_eq!(self.len(), 1); + Fields::from_single_pattern(subpattern) + } + PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => { + self.replace_with_fieldpats(subpatterns) + } + PatKind::Array { prefix, suffix, .. } | PatKind::Slice { prefix, suffix, .. } => { + // Number of subpatterns for the constructor + let ctor_arity = self.len(); + + // Replace the prefix and the suffix with the given patterns, leaving wildcards in + // the middle if there was a subslice pattern `..`. + let prefix = prefix.iter().enumerate(); + let suffix = + suffix.iter().enumerate().map(|(i, p)| (ctor_arity - suffix.len() + i, p)); + self.replace_fields_indexed(prefix.chain(suffix)) + } + _ => self.clone(), + } + } + fn push_on_patstack(self, stack: &[&'p Pat<'tcx>]) -> PatStack<'p, 'tcx> { let pats: SmallVec<_> = match self { Fields::Slice(pats) => pats.iter().chain(stack.iter().copied()).collect(), @@ -1401,60 +1377,21 @@ impl<'tcx> Usefulness<'tcx> { } fn is_useful(&self) -> bool { - match *self { - NotUseful => false, - _ => true, - } + !matches!(*self, NotUseful) } fn apply_constructor<'p>( self, - cx: &MatchCheckCtxt<'p, 'tcx>, + pcx: PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>, - ty: Ty<'tcx>, ctor_wild_subpatterns: &Fields<'p, 'tcx>, + is_top_level: bool, ) -> Self { match self { - UsefulWithWitness(witnesses) => UsefulWithWitness( - witnesses - .into_iter() - .map(|witness| witness.apply_constructor(cx, &ctor, ty, ctor_wild_subpatterns)) - .collect(), - ), - x => x, - } - } - - fn apply_wildcard(self, ty: Ty<'tcx>) -> Self { - match self { UsefulWithWitness(witnesses) => { - let wild = Pat::wildcard_from_ty(ty); - UsefulWithWitness( - witnesses - .into_iter() - .map(|mut witness| { - witness.0.push(wild.clone()); - witness - }) - .collect(), - ) - } - x => x, - } - } - - fn apply_missing_ctors( - self, - cx: &MatchCheckCtxt<'_, 'tcx>, - ty: Ty<'tcx>, - missing_ctors: &MissingConstructors<'tcx>, - ) -> Self { - match self { - UsefulWithWitness(witnesses) => { - let new_patterns: Vec<_> = - missing_ctors.iter().map(|ctor| ctor.apply_wildcards(cx, ty)).collect(); - // Add the new patterns to each witness - UsefulWithWitness( + let new_witnesses = if ctor.is_wildcard() { + let missing_ctors = MissingConstructors::new(pcx); + let new_patterns = missing_ctors.report_patterns(pcx, is_top_level); witnesses .into_iter() .flat_map(|witness| { @@ -1464,8 +1401,14 @@ impl<'tcx> Usefulness<'tcx> { witness }) }) - .collect(), - ) + .collect() + } else { + witnesses + .into_iter() + .map(|witness| witness.apply_constructor(pcx, &ctor, ctor_wild_subpatterns)) + .collect() + }; + UsefulWithWitness(new_witnesses) } x => x, } @@ -1478,9 +1421,14 @@ crate enum WitnessPreference { LeaveOutWitness, } -#[derive(Copy, Clone, Debug)] -struct PatCtxt<'tcx> { +#[derive(Copy, Clone)] +struct PatCtxt<'a, 'p, 'tcx> { + cx: &'a MatchCheckCtxt<'p, 'tcx>, + /// Current state of the matrix. + matrix: &'a Matrix<'p, 'tcx>, + /// Type of the current column under investigation. ty: Ty<'tcx>, + /// Span of the current pattern under investigation. span: Span, } @@ -1496,6 +1444,7 @@ struct PatCtxt<'tcx> { /// multiple patterns. /// /// For example, if we are constructing a witness for the match against +/// /// ``` /// struct Pair(Option<(u32, u32)>, bool); /// @@ -1540,17 +1489,16 @@ impl<'tcx> Witness<'tcx> { /// pats: [(false, "foo"), 42] => X { a: (false, "foo"), b: 42 } fn apply_constructor<'p>( mut self, - cx: &MatchCheckCtxt<'p, 'tcx>, + pcx: PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>, - ty: Ty<'tcx>, ctor_wild_subpatterns: &Fields<'p, 'tcx>, ) -> Self { let pat = { let len = self.0.len(); let arity = ctor_wild_subpatterns.len(); let pats = self.0.drain((len - arity)..).rev(); - let fields = ctor_wild_subpatterns.replace_fields(cx, pats); - ctor.apply(cx, ty, fields) + let fields = ctor_wild_subpatterns.replace_fields(pcx.cx, pats); + ctor.apply(pcx, fields) }; self.0.push(pat); @@ -1568,11 +1516,9 @@ impl<'tcx> Witness<'tcx> { /// `Option<!>`, we do not include `Some(_)` in the returned list of constructors. /// Invariant: this returns an empty `Vec` if and only if the type is uninhabited (as determined by /// `cx.is_uninhabited()`). -fn all_constructors<'a, 'tcx>( - cx: &mut MatchCheckCtxt<'a, 'tcx>, - pcx: PatCtxt<'tcx>, -) -> Vec<Constructor<'tcx>> { +fn all_constructors<'p, 'tcx>(pcx: PatCtxt<'_, 'p, 'tcx>) -> Vec<Constructor<'tcx>> { debug!("all_constructors({:?})", pcx.ty); + let cx = pcx.cx; let make_range = |start, end| { IntRange( // `unwrap()` is ok because we know the type is an integer. @@ -1580,51 +1526,36 @@ fn all_constructors<'a, 'tcx>( .unwrap(), ) }; - match *pcx.ty.kind() { - ty::Bool => { - [true, false].iter().map(|&b| ConstantValue(ty::Const::from_bool(cx.tcx, b))).collect() - } - ty::Array(ref sub_ty, len) if len.try_eval_usize(cx.tcx, cx.param_env).is_some() => { + match pcx.ty.kind() { + ty::Bool => vec![make_range(0, 1)], + ty::Array(sub_ty, len) if len.try_eval_usize(cx.tcx, cx.param_env).is_some() => { let len = len.eval_usize(cx.tcx, cx.param_env); if len != 0 && cx.is_uninhabited(sub_ty) { vec![] } else { - vec![Slice(Slice { array_len: Some(len), kind: VarLen(0, 0) })] + vec![Slice(Slice::new(Some(len), VarLen(0, 0)))] } } // Treat arrays of a constant but unknown length like slices. - ty::Array(ref sub_ty, _) | ty::Slice(ref sub_ty) => { + ty::Array(sub_ty, _) | ty::Slice(sub_ty) => { let kind = if cx.is_uninhabited(sub_ty) { FixedLen(0) } else { VarLen(0, 0) }; - vec![Slice(Slice { array_len: None, kind })] + vec![Slice(Slice::new(None, kind))] } ty::Adt(def, substs) if def.is_enum() => { - let ctors: Vec<_> = if cx.tcx.features().exhaustive_patterns { - // If `exhaustive_patterns` is enabled, we exclude variants known to be - // uninhabited. - def.variants - .iter() - .filter(|v| { - !v.uninhabited_from(cx.tcx, substs, def.adt_kind(), cx.param_env) - .contains(cx.tcx, cx.module) - }) - .map(|v| Variant(v.def_id)) - .collect() - } else { - def.variants.iter().map(|v| Variant(v.def_id)).collect() - }; - // If the enum is declared as `#[non_exhaustive]`, we treat it as if it had an // additional "unknown" constructor. // There is no point in enumerating all possible variants, because the user can't // actually match against them all themselves. So we always return only the fictitious // constructor. // E.g., in an example like: + // // ``` // let err: io::ErrorKind = ...; // match err { // io::ErrorKind::NotFound => {}, // } // ``` + // // we don't want to show every possible IO error, but instead have only `_` as the // witness. let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(pcx.ty); @@ -1636,7 +1567,22 @@ fn all_constructors<'a, 'tcx>( let is_secretly_empty = def.variants.is_empty() && !cx.tcx.features().exhaustive_patterns; - if is_secretly_empty || is_declared_nonexhaustive { vec![NonExhaustive] } else { ctors } + if is_secretly_empty || is_declared_nonexhaustive { + vec![NonExhaustive] + } else if cx.tcx.features().exhaustive_patterns { + // If `exhaustive_patterns` is enabled, we exclude variants known to be + // uninhabited. + def.variants + .iter() + .filter(|v| { + !v.uninhabited_from(cx.tcx, substs, def.adt_kind(), cx.param_env) + .contains(cx.tcx, cx.module) + }) + .map(|v| Variant(v.def_id)) + .collect() + } else { + def.variants.iter().map(|v| Variant(v.def_id)).collect() + } } ty::Char => { vec![ @@ -1654,24 +1600,21 @@ fn all_constructors<'a, 'tcx>( // `#[non_exhaustive]` enums by returning a special unmatcheable constructor. vec![NonExhaustive] } - ty::Int(ity) => { + &ty::Int(ity) => { let bits = Integer::from_attr(&cx.tcx, SignedInt(ity)).size().bits() as u128; let min = 1u128 << (bits - 1); let max = min - 1; vec![make_range(min, max)] } - ty::Uint(uty) => { + &ty::Uint(uty) => { let size = Integer::from_attr(&cx.tcx, UnsignedInt(uty)).size(); let max = truncate(u128::MAX, size); vec![make_range(0, max)] } - _ => { - if cx.is_uninhabited(pcx.ty) { - vec![] - } else { - vec![Single] - } - } + _ if cx.is_uninhabited(pcx.ty) => vec![], + ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => vec![Single], + // This type is one for which we cannot list constructors, like `str` or `f64`. + _ => vec![NonExhaustive], } } @@ -1695,10 +1638,7 @@ struct IntRange<'tcx> { impl<'tcx> IntRange<'tcx> { #[inline] fn is_integral(ty: Ty<'_>) -> bool { - match ty.kind() { - ty::Char | ty::Int(_) | ty::Uint(_) => true, - _ => false, - } + matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_) | ty::Bool) } fn is_singleton(&self) -> bool { @@ -1718,6 +1658,7 @@ impl<'tcx> IntRange<'tcx> { #[inline] fn integral_size_and_signed_bias(tcx: TyCtxt<'tcx>, ty: Ty<'_>) -> Option<(Size, u128)> { match *ty.kind() { + ty::Bool => Some((Size::from_bytes(1), 0)), ty::Char => Some((Size::from_bytes(4), 0)), ty::Int(ity) => { let size = Integer::from_attr(&tcx, SignedInt(ity)).size(); @@ -1782,40 +1723,6 @@ impl<'tcx> IntRange<'tcx> { } } - fn from_pat( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - pat: &Pat<'tcx>, - ) -> Option<IntRange<'tcx>> { - // This MUST be kept in sync with `pat_constructor`. - match *pat.kind { - PatKind::AscribeUserType { .. } => bug!(), // Handled by `expand_pattern` - PatKind::Or { .. } => bug!("Or-pattern should have been expanded earlier on."), - - PatKind::Binding { .. } - | PatKind::Wild - | PatKind::Leaf { .. } - | PatKind::Deref { .. } - | PatKind::Variant { .. } - | PatKind::Array { .. } - | PatKind::Slice { .. } => None, - - PatKind::Constant { value } => Self::from_const(tcx, param_env, value, pat.span), - - PatKind::Range(PatRange { lo, hi, end }) => { - let ty = lo.ty; - Self::from_range( - tcx, - lo.eval_bits(tcx, param_env, lo.ty), - hi.eval_bits(tcx, param_env, hi.ty), - ty, - &end, - pat.span, - ) - } - } - } - // The return value of `signed_bias` should be XORed with an endpoint to encode/decode it. fn signed_bias(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> u128 { match *ty.kind() { @@ -1827,35 +1734,6 @@ impl<'tcx> IntRange<'tcx> { } } - /// Returns a collection of ranges that spans the values covered by `ranges`, subtracted - /// by the values covered by `self`: i.e., `ranges \ self` (in set notation). - fn subtract_from(&self, ranges: Vec<IntRange<'tcx>>) -> Vec<IntRange<'tcx>> { - let mut remaining_ranges = vec![]; - let ty = self.ty; - let span = self.span; - let (lo, hi) = self.boundaries(); - for subrange in ranges { - let (subrange_lo, subrange_hi) = subrange.range.into_inner(); - if lo > subrange_hi || subrange_lo > hi { - // The pattern doesn't intersect with the subrange at all, - // so the subrange remains untouched. - remaining_ranges.push(IntRange { range: subrange_lo..=subrange_hi, ty, span }); - } else { - if lo > subrange_lo { - // The pattern intersects an upper section of the - // subrange, so a lower section will remain. - remaining_ranges.push(IntRange { range: subrange_lo..=(lo - 1), ty, span }); - } - if hi < subrange_hi { - // The pattern intersects a lower section of the - // subrange, so an upper section will remain. - remaining_ranges.push(IntRange { range: (hi + 1)..=subrange_hi, ty, span }); - } - } - } - remaining_ranges - } - fn is_subrange(&self, other: &Self) -> bool { other.range.start() <= self.range.start() && self.range.end() <= other.range.end() } @@ -1913,6 +1791,162 @@ impl<'tcx> IntRange<'tcx> { // This is a brand new pattern, so we don't reuse `self.span`. Pat { ty: self.ty, span: DUMMY_SP, kind: Box::new(kind) } } + + /// For exhaustive integer matching, some constructors are grouped within other constructors + /// (namely integer typed values are grouped within ranges). However, when specialising these + /// constructors, we want to be specialising for the underlying constructors (the integers), not + /// the groups (the ranges). Thus we need to split the groups up. Splitting them up naïvely would + /// mean creating a separate constructor for every single value in the range, which is clearly + /// impractical. However, observe that for some ranges of integers, the specialisation will be + /// identical across all values in that range (i.e., there are equivalence classes of ranges of + /// constructors based on their `U(S(c, P), S(c, p))` outcome). These classes are grouped by + /// the patterns that apply to them (in the matrix `P`). We can split the range whenever the + /// patterns that apply to that range (specifically: the patterns that *intersect* with that range) + /// change. + /// Our solution, therefore, is to split the range constructor into subranges at every single point + /// the group of intersecting patterns changes (using the method described below). + /// And voilà ! We're testing precisely those ranges that we need to, without any exhaustive matching + /// on actual integers. The nice thing about this is that the number of subranges is linear in the + /// number of rows in the matrix (i.e., the number of cases in the `match` statement), so we don't + /// need to be worried about matching over gargantuan ranges. + /// + /// Essentially, given the first column of a matrix representing ranges, looking like the following: + /// + /// |------| |----------| |-------| || + /// |-------| |-------| |----| || + /// |---------| + /// + /// We split the ranges up into equivalence classes so the ranges are no longer overlapping: + /// + /// |--|--|||-||||--||---|||-------| |-|||| || + /// + /// The logic for determining how to split the ranges is fairly straightforward: we calculate + /// boundaries for each interval range, sort them, then create constructors for each new interval + /// between every pair of boundary points. (This essentially sums up to performing the intuitive + /// merging operation depicted above.) + fn split<'p>( + &self, + pcx: PatCtxt<'_, 'p, 'tcx>, + hir_id: Option<HirId>, + ) -> SmallVec<[Constructor<'tcx>; 1]> { + let ty = pcx.ty; + + /// Represents a border between 2 integers. Because the intervals spanning borders + /// must be able to cover every integer, we need to be able to represent + /// 2^128 + 1 such borders. + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] + enum Border { + JustBefore(u128), + AfterMax, + } + + // A function for extracting the borders of an integer interval. + fn range_borders(r: IntRange<'_>) -> impl Iterator<Item = Border> { + let (lo, hi) = r.range.into_inner(); + let from = Border::JustBefore(lo); + let to = match hi.checked_add(1) { + Some(m) => Border::JustBefore(m), + None => Border::AfterMax, + }; + vec![from, to].into_iter() + } + + // Collect the span and range of all the intersecting ranges to lint on likely + // incorrect range patterns. (#63987) + let mut overlaps = vec![]; + let row_len = pcx.matrix.patterns.get(0).map(|r| r.len()).unwrap_or(0); + // `borders` is the set of borders between equivalence classes: each equivalence + // class lies between 2 borders. + let row_borders = pcx + .matrix + .head_ctors(pcx.cx) + .filter_map(|ctor| ctor.as_int_range()) + .filter_map(|range| { + let intersection = self.intersection(pcx.cx.tcx, &range); + let should_lint = self.suspicious_intersection(&range); + if let (Some(range), 1, true) = (&intersection, row_len, should_lint) { + // FIXME: for now, only check for overlapping ranges on simple range + // patterns. Otherwise with the current logic the following is detected + // as overlapping: + // match (10u8, true) { + // (0 ..= 125, false) => {} + // (126 ..= 255, false) => {} + // (0 ..= 255, true) => {} + // } + overlaps.push(range.clone()); + } + intersection + }) + .flat_map(range_borders); + let self_borders = range_borders(self.clone()); + let mut borders: Vec<_> = row_borders.chain(self_borders).collect(); + borders.sort_unstable(); + + self.lint_overlapping_patterns(pcx.cx.tcx, hir_id, ty, overlaps); + + // We're going to iterate through every adjacent pair of borders, making sure that + // each represents an interval of nonnegative length, and convert each such + // interval into a constructor. + borders + .array_windows() + .filter_map(|&pair| match pair { + [Border::JustBefore(n), Border::JustBefore(m)] => { + if n < m { + Some(n..=(m - 1)) + } else { + None + } + } + [Border::JustBefore(n), Border::AfterMax] => Some(n..=u128::MAX), + [Border::AfterMax, _] => None, + }) + .map(|range| IntRange { range, ty, span: pcx.span }) + .map(IntRange) + .collect() + } + + fn lint_overlapping_patterns( + &self, + tcx: TyCtxt<'tcx>, + hir_id: Option<HirId>, + ty: Ty<'tcx>, + overlaps: Vec<IntRange<'tcx>>, + ) { + if let (true, Some(hir_id)) = (!overlaps.is_empty(), hir_id) { + tcx.struct_span_lint_hir( + lint::builtin::OVERLAPPING_PATTERNS, + hir_id, + self.span, + |lint| { + let mut err = lint.build("multiple patterns covering the same range"); + err.span_label(self.span, "overlapping patterns"); + for int_range in overlaps { + // Use the real type for user display of the ranges: + err.span_label( + int_range.span, + &format!( + "this range overlaps on `{}`", + IntRange { range: int_range.range, ty, span: DUMMY_SP }.to_pat(tcx), + ), + ); + } + err.emit(); + }, + ); + } + } + + /// See `Constructor::is_covered_by` + fn is_covered_by<'p>(&self, pcx: PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool { + if self.intersection(pcx.cx.tcx, other).is_some() { + // Constructor splitting should ensure that all intersections we encounter are actually + // inclusions. + assert!(self.is_subrange(other)); + true + } else { + false + } + } } /// Ignore spans when comparing, they don't carry semantic information as they are only for lints. @@ -1923,39 +1957,86 @@ impl<'tcx> std::cmp::PartialEq for IntRange<'tcx> { } // A struct to compute a set of constructors equivalent to `all_ctors \ used_ctors`. +#[derive(Debug)] struct MissingConstructors<'tcx> { - all_ctors: Vec<Constructor<'tcx>>, + all_ctors: SmallVec<[Constructor<'tcx>; 1]>, used_ctors: Vec<Constructor<'tcx>>, } impl<'tcx> MissingConstructors<'tcx> { - fn new(all_ctors: Vec<Constructor<'tcx>>, used_ctors: Vec<Constructor<'tcx>>) -> Self { - MissingConstructors { all_ctors, used_ctors } - } + fn new<'p>(pcx: PatCtxt<'_, 'p, 'tcx>) -> Self { + let used_ctors: Vec<Constructor<'_>> = + pcx.matrix.head_ctors(pcx.cx).cloned().filter(|c| !c.is_wildcard()).collect(); + // Since `all_ctors` never contains wildcards, this won't recurse further. + let all_ctors = + all_constructors(pcx).into_iter().flat_map(|ctor| ctor.split(pcx, None)).collect(); - fn into_inner(self) -> (Vec<Constructor<'tcx>>, Vec<Constructor<'tcx>>) { - (self.all_ctors, self.used_ctors) + MissingConstructors { all_ctors, used_ctors } } - fn is_empty(&self) -> bool { - self.iter().next().is_none() - } - /// Whether this contains all the constructors for the given type or only a - /// subset. - fn all_ctors_are_missing(&self) -> bool { - self.used_ctors.is_empty() + fn is_empty<'p>(&self, pcx: PatCtxt<'_, 'p, 'tcx>) -> bool { + self.iter(pcx).next().is_none() } /// Iterate over all_ctors \ used_ctors - fn iter<'a>(&'a self) -> impl Iterator<Item = Constructor<'tcx>> + Captures<'a> { - self.all_ctors.iter().flat_map(move |req_ctor| req_ctor.subtract_ctors(&self.used_ctors)) + fn iter<'a, 'p>( + &'a self, + pcx: PatCtxt<'a, 'p, 'tcx>, + ) -> impl Iterator<Item = &'a Constructor<'tcx>> + Captures<'p> { + self.all_ctors.iter().filter(move |ctor| !ctor.is_covered_by_any(pcx, &self.used_ctors)) } -} -impl<'tcx> fmt::Debug for MissingConstructors<'tcx> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let ctors: Vec<_> = self.iter().collect(); - write!(f, "{:?}", ctors) + /// List the patterns corresponding to the missing constructors. In some cases, instead of + /// listing all constructors of a given type, we prefer to simply report a wildcard. + fn report_patterns<'p>( + &self, + pcx: PatCtxt<'_, 'p, 'tcx>, + is_top_level: bool, + ) -> SmallVec<[Pat<'tcx>; 1]> { + // There are 2 ways we can report a witness here. + // Commonly, we can report all the "free" + // constructors as witnesses, e.g., if we have: + // + // ``` + // enum Direction { N, S, E, W } + // let Direction::N = ...; + // ``` + // + // we can report 3 witnesses: `S`, `E`, and `W`. + // + // However, there is a case where we don't want + // to do this and instead report a single `_` witness: + // if the user didn't actually specify a constructor + // in this arm, e.g., in + // + // ``` + // let x: (Direction, Direction, bool) = ...; + // let (_, _, false) = x; + // ``` + // + // we don't want to show all 16 possible witnesses + // `(<direction-1>, <direction-2>, true)` - we are + // satisfied with `(_, _, true)`. In this case, + // `used_ctors` is empty. + // The exception is: if we are at the top-level, for example in an empty match, we + // sometimes prefer reporting the list of constructors instead of just `_`. + let report_when_all_missing = is_top_level && !IntRange::is_integral(pcx.ty); + if self.used_ctors.is_empty() && !report_when_all_missing { + // All constructors are unused. Report only a wildcard + // rather than each individual constructor. + smallvec![Pat::wildcard_from_ty(pcx.ty)] + } else { + // Construct for each missing constructor a "wild" version of this + // constructor, that matches everything that can be built with + // it. For example, if `ctor` is a `Constructor::Variant` for + // `Option::Some`, we get the pattern `Some(_)`. + self.iter(pcx) + .map(|missing_ctor| { + let fields = Fields::wildcards(pcx, &missing_ctor); + missing_ctor.apply(pcx, fields) + }) + .collect() + } } } @@ -1982,7 +2063,7 @@ impl<'tcx> fmt::Debug for MissingConstructors<'tcx> { /// has one it must not be inserted into the matrix. This shouldn't be /// relied on for soundness. crate fn is_useful<'p, 'tcx>( - cx: &mut MatchCheckCtxt<'p, 'tcx>, + cx: &MatchCheckCtxt<'p, 'tcx>, matrix: &Matrix<'p, 'tcx>, v: &PatStack<'p, 'tcx>, witness_preference: WitnessPreference, @@ -2017,6 +2098,7 @@ crate fn is_useful<'p, 'tcx>( let mut unreachable_branches = Vec::new(); // Subpatterns that are unreachable from all branches. E.g. in the following case, the last // `true` is unreachable only from one branch, so it is overall reachable. + // // ``` // match (true, true) { // (true, true) => {} @@ -2062,209 +2144,77 @@ crate fn is_useful<'p, 'tcx>( // FIXME(Nadrieril): Hack to work around type normalization issues (see #72476). let ty = matrix.heads().next().map(|r| r.ty).unwrap_or(v.head().ty); - let pcx = PatCtxt { ty, span: v.head().span }; - - debug!("is_useful_expand_first_col: pcx={:#?}, expanding {:#?}", pcx, v.head()); - - let ret = if let Some(constructor) = pat_constructor(cx.tcx, cx.param_env, v.head()) { - debug!("is_useful - expanding constructor: {:#?}", constructor); - split_grouped_constructors( - cx.tcx, - cx.param_env, - pcx, - vec![constructor], - matrix, - pcx.span, - Some(hir_id), - ) - .into_iter() - .map(|c| { - is_useful_specialized( - cx, - matrix, - v, - c, - pcx.ty, - witness_preference, - hir_id, - is_under_guard, - ) - }) - .find(|result| result.is_useful()) - .unwrap_or(NotUseful) - } else { - debug!("is_useful - expanding wildcard"); + let pcx = PatCtxt { cx, matrix, ty, span: v.head().span }; - let used_ctors: Vec<Constructor<'_>> = - matrix.heads().filter_map(|p| pat_constructor(cx.tcx, cx.param_env, p)).collect(); - debug!("is_useful_used_ctors = {:#?}", used_ctors); - // `all_ctors` are all the constructors for the given type, which - // should all be represented (or caught with the wild pattern `_`). - let all_ctors = all_constructors(cx, pcx); - debug!("is_useful_all_ctors = {:#?}", all_ctors); - - // `missing_ctors` is the set of constructors from the same type as the - // first column of `matrix` that are matched only by wildcard patterns - // from the first column. - // - // Therefore, if there is some pattern that is unmatched by `matrix`, - // it will still be unmatched if the first constructor is replaced by - // any of the constructors in `missing_ctors` + debug!("is_useful_expand_first_col: ty={:#?}, expanding {:#?}", pcx.ty, v.head()); - // Missing constructors are those that are not matched by any non-wildcard patterns in the - // current column. We only fully construct them on-demand, because they're rarely used and - // can be big. - let missing_ctors = MissingConstructors::new(all_ctors, used_ctors); - - debug!("is_useful_missing_ctors.empty()={:#?}", missing_ctors.is_empty(),); - - if missing_ctors.is_empty() { - let (all_ctors, _) = missing_ctors.into_inner(); - split_grouped_constructors(cx.tcx, cx.param_env, pcx, all_ctors, matrix, DUMMY_SP, None) - .into_iter() - .map(|c| { - is_useful_specialized( - cx, - matrix, - v, - c, - pcx.ty, - witness_preference, - hir_id, - is_under_guard, - ) - }) - .find(|result| result.is_useful()) - .unwrap_or(NotUseful) - } else { - let matrix = matrix.specialize_wildcard(); - let v = v.to_tail(); + let ret = v + .head_ctor(cx) + .split(pcx, Some(hir_id)) + .into_iter() + .map(|ctor| { + // We cache the result of `Fields::wildcards` because it is used a lot. + let ctor_wild_subpatterns = Fields::wildcards(pcx, &ctor); + let matrix = pcx.matrix.specialize_constructor(pcx, &ctor, &ctor_wild_subpatterns); + let v = v.pop_head_constructor(&ctor_wild_subpatterns); let usefulness = - is_useful(cx, &matrix, &v, witness_preference, hir_id, is_under_guard, false); - - // In this case, there's at least one "free" - // constructor that is only matched against by - // wildcard patterns. - // - // There are 2 ways we can report a witness here. - // Commonly, we can report all the "free" - // constructors as witnesses, e.g., if we have: - // - // ``` - // enum Direction { N, S, E, W } - // let Direction::N = ...; - // ``` - // - // we can report 3 witnesses: `S`, `E`, and `W`. - // - // However, there is a case where we don't want - // to do this and instead report a single `_` witness: - // if the user didn't actually specify a constructor - // in this arm, e.g., in - // ``` - // let x: (Direction, Direction, bool) = ...; - // let (_, _, false) = x; - // ``` - // we don't want to show all 16 possible witnesses - // `(<direction-1>, <direction-2>, true)` - we are - // satisfied with `(_, _, true)`. In this case, - // `used_ctors` is empty. - // The exception is: if we are at the top-level, for example in an empty match, we - // sometimes prefer reporting the list of constructors instead of just `_`. - let report_ctors_rather_than_wildcard = is_top_level && !IntRange::is_integral(pcx.ty); - if missing_ctors.all_ctors_are_missing() && !report_ctors_rather_than_wildcard { - // All constructors are unused. Add a wild pattern - // rather than each individual constructor. - usefulness.apply_wildcard(pcx.ty) - } else { - // Construct for each missing constructor a "wild" version of this - // constructor, that matches everything that can be built with - // it. For example, if `ctor` is a `Constructor::Variant` for - // `Option::Some`, we get the pattern `Some(_)`. - usefulness.apply_missing_ctors(cx, pcx.ty, &missing_ctors) - } - } - }; + is_useful(pcx.cx, &matrix, &v, witness_preference, hir_id, is_under_guard, false); + usefulness.apply_constructor(pcx, &ctor, &ctor_wild_subpatterns, is_top_level) + }) + .find(|result| result.is_useful()) + .unwrap_or(NotUseful); debug!("is_useful::returns({:#?}, {:#?}) = {:?}", matrix, v, ret); ret } -/// A shorthand for the `U(S(c, P), S(c, q))` operation from the paper. I.e., `is_useful` applied -/// to the specialised version of both the pattern matrix `P` and the new pattern `q`. -fn is_useful_specialized<'p, 'tcx>( - cx: &mut MatchCheckCtxt<'p, 'tcx>, - matrix: &Matrix<'p, 'tcx>, - v: &PatStack<'p, 'tcx>, - ctor: Constructor<'tcx>, - ty: Ty<'tcx>, - witness_preference: WitnessPreference, - hir_id: HirId, - is_under_guard: bool, -) -> Usefulness<'tcx> { - debug!("is_useful_specialized({:#?}, {:#?}, {:?})", v, ctor, ty); - - // We cache the result of `Fields::wildcards` because it is used a lot. - let ctor_wild_subpatterns = Fields::wildcards(cx, &ctor, ty); - let matrix = matrix.specialize_constructor(cx, &ctor, &ctor_wild_subpatterns); - v.specialize_constructor(cx, &ctor, &ctor_wild_subpatterns) - .map(|v| is_useful(cx, &matrix, &v, witness_preference, hir_id, is_under_guard, false)) - .map(|u| u.apply_constructor(cx, &ctor, ty, &ctor_wild_subpatterns)) - .unwrap_or(NotUseful) -} - /// Determines the constructor that the given pattern can be specialized to. /// Returns `None` in case of a catch-all, which can't be specialized. -fn pat_constructor<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - pat: &Pat<'tcx>, -) -> Option<Constructor<'tcx>> { - // This MUST be kept in sync with `IntRange::from_pat`. - match *pat.kind { +fn pat_constructor<'p, 'tcx>( + cx: &MatchCheckCtxt<'p, 'tcx>, + pat: &'p Pat<'tcx>, +) -> Constructor<'tcx> { + match pat.kind.as_ref() { PatKind::AscribeUserType { .. } => bug!(), // Handled by `expand_pattern` - PatKind::Binding { .. } | PatKind::Wild => None, - PatKind::Leaf { .. } | PatKind::Deref { .. } => Some(Single), - PatKind::Variant { adt_def, variant_index, .. } => { - Some(Variant(adt_def.variants[variant_index].def_id)) + PatKind::Binding { .. } | PatKind::Wild => Wildcard, + PatKind::Leaf { .. } | PatKind::Deref { .. } => Single, + &PatKind::Variant { adt_def, variant_index, .. } => { + Variant(adt_def.variants[variant_index].def_id) } PatKind::Constant { value } => { - if let Some(int_range) = IntRange::from_const(tcx, param_env, value, pat.span) { - Some(IntRange(int_range)) + if let Some(int_range) = IntRange::from_const(cx.tcx, cx.param_env, value, pat.span) { + IntRange(int_range) } else { - match (value.val, &value.ty.kind()) { - (_, ty::Array(_, n)) => { - let len = n.eval_usize(tcx, param_env); - Some(Slice(Slice { array_len: Some(len), kind: FixedLen(len) })) - } - (ty::ConstKind::Value(ConstValue::Slice { start, end, .. }), ty::Slice(_)) => { - let len = (end - start) as u64; - Some(Slice(Slice { array_len: None, kind: FixedLen(len) })) - } - // FIXME(oli-obk): implement `deref` for `ConstValue` - // (ty::ConstKind::Value(ConstValue::ByRef { .. }), ty::Slice(_)) => { ... } - _ => Some(ConstantValue(value)), + match pat.ty.kind() { + ty::Float(_) => FloatRange(value, value, RangeEnd::Included), + // In `expand_pattern`, we convert string literals to `&CONST` patterns with + // `CONST` a pattern of type `str`. In truth this contains a constant of type + // `&str`. + ty::Str => Str(value), + // All constants that can be structurally matched have already been expanded + // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are + // opaque. + _ => Opaque, } } } - PatKind::Range(PatRange { lo, hi, end }) => { + &PatKind::Range(PatRange { lo, hi, end }) => { let ty = lo.ty; if let Some(int_range) = IntRange::from_range( - tcx, - lo.eval_bits(tcx, param_env, lo.ty), - hi.eval_bits(tcx, param_env, hi.ty), + cx.tcx, + lo.eval_bits(cx.tcx, cx.param_env, lo.ty), + hi.eval_bits(cx.tcx, cx.param_env, hi.ty), ty, &end, pat.span, ) { - Some(IntRange(int_range)) + IntRange(int_range) } else { - Some(FloatRange(lo, hi, end)) + FloatRange(lo, hi, end) } } - PatKind::Array { ref prefix, ref slice, ref suffix } - | PatKind::Slice { ref prefix, ref slice, ref suffix } => { + PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => { let array_len = match pat.ty.kind() { - ty::Array(_, length) => Some(length.eval_usize(tcx, param_env)), + ty::Array(_, length) => Some(length.eval_usize(cx.tcx, cx.param_env)), ty::Slice(_) => None, _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty), }; @@ -2272,584 +2222,8 @@ fn pat_constructor<'tcx>( let suffix = suffix.len() as u64; let kind = if slice.is_some() { VarLen(prefix, suffix) } else { FixedLen(prefix + suffix) }; - Some(Slice(Slice { array_len, kind })) + Slice(Slice::new(array_len, kind)) } PatKind::Or { .. } => bug!("Or-pattern should have been expanded earlier on."), } } - -// checks whether a constant is equal to a user-written slice pattern. Only supports byte slices, -// meaning all other types will compare unequal and thus equal patterns often do not cause the -// second pattern to lint about unreachable match arms. -fn slice_pat_covered_by_const<'tcx>( - tcx: TyCtxt<'tcx>, - _span: Span, - const_val: &'tcx ty::Const<'tcx>, - prefix: &[Pat<'tcx>], - slice: &Option<Pat<'tcx>>, - suffix: &[Pat<'tcx>], - param_env: ty::ParamEnv<'tcx>, -) -> Result<bool, ErrorReported> { - let const_val_val = if let ty::ConstKind::Value(val) = const_val.val { - val - } else { - bug!( - "slice_pat_covered_by_const: {:#?}, {:#?}, {:#?}, {:#?}", - const_val, - prefix, - slice, - suffix, - ) - }; - - let data: &[u8] = match (const_val_val, &const_val.ty.kind()) { - (ConstValue::ByRef { offset, alloc, .. }, ty::Array(t, n)) => { - assert_eq!(*t, tcx.types.u8); - let n = n.eval_usize(tcx, param_env); - let ptr = Pointer::new(AllocId(0), offset); - alloc.get_bytes(&tcx, ptr, Size::from_bytes(n)).unwrap() - } - (ConstValue::Slice { data, start, end }, ty::Slice(t)) => { - assert_eq!(*t, tcx.types.u8); - let ptr = Pointer::new(AllocId(0), Size::from_bytes(start)); - data.get_bytes(&tcx, ptr, Size::from_bytes(end - start)).unwrap() - } - // FIXME(oli-obk): create a way to extract fat pointers from ByRef - (_, ty::Slice(_)) => return Ok(false), - _ => bug!( - "slice_pat_covered_by_const: {:#?}, {:#?}, {:#?}, {:#?}", - const_val, - prefix, - slice, - suffix, - ), - }; - - let pat_len = prefix.len() + suffix.len(); - if data.len() < pat_len || (slice.is_none() && data.len() > pat_len) { - return Ok(false); - } - - for (ch, pat) in data[..prefix.len()] - .iter() - .zip(prefix) - .chain(data[data.len() - suffix.len()..].iter().zip(suffix)) - { - if let box PatKind::Constant { value } = pat.kind { - let b = value.eval_bits(tcx, param_env, pat.ty); - assert_eq!(b as u8 as u128, b); - if b as u8 != *ch { - return Ok(false); - } - } - } - - Ok(true) -} - -/// For exhaustive integer matching, some constructors are grouped within other constructors -/// (namely integer typed values are grouped within ranges). However, when specialising these -/// constructors, we want to be specialising for the underlying constructors (the integers), not -/// the groups (the ranges). Thus we need to split the groups up. Splitting them up naïvely would -/// mean creating a separate constructor for every single value in the range, which is clearly -/// impractical. However, observe that for some ranges of integers, the specialisation will be -/// identical across all values in that range (i.e., there are equivalence classes of ranges of -/// constructors based on their `is_useful_specialized` outcome). These classes are grouped by -/// the patterns that apply to them (in the matrix `P`). We can split the range whenever the -/// patterns that apply to that range (specifically: the patterns that *intersect* with that range) -/// change. -/// Our solution, therefore, is to split the range constructor into subranges at every single point -/// the group of intersecting patterns changes (using the method described below). -/// And voilà ! We're testing precisely those ranges that we need to, without any exhaustive matching -/// on actual integers. The nice thing about this is that the number of subranges is linear in the -/// number of rows in the matrix (i.e., the number of cases in the `match` statement), so we don't -/// need to be worried about matching over gargantuan ranges. -/// -/// Essentially, given the first column of a matrix representing ranges, looking like the following: -/// -/// |------| |----------| |-------| || -/// |-------| |-------| |----| || -/// |---------| -/// -/// We split the ranges up into equivalence classes so the ranges are no longer overlapping: -/// -/// |--|--|||-||||--||---|||-------| |-|||| || -/// -/// The logic for determining how to split the ranges is fairly straightforward: we calculate -/// boundaries for each interval range, sort them, then create constructors for each new interval -/// between every pair of boundary points. (This essentially sums up to performing the intuitive -/// merging operation depicted above.) -/// -/// `hir_id` is `None` when we're evaluating the wildcard pattern, do not lint for overlapping in -/// ranges that case. -/// -/// This also splits variable-length slices into fixed-length slices. -fn split_grouped_constructors<'p, 'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - pcx: PatCtxt<'tcx>, - ctors: Vec<Constructor<'tcx>>, - matrix: &Matrix<'p, 'tcx>, - span: Span, - hir_id: Option<HirId>, -) -> Vec<Constructor<'tcx>> { - let ty = pcx.ty; - let mut split_ctors = Vec::with_capacity(ctors.len()); - debug!("split_grouped_constructors({:#?}, {:#?})", matrix, ctors); - - for ctor in ctors.into_iter() { - match ctor { - IntRange(ctor_range) if ctor_range.treat_exhaustively(tcx) => { - // Fast-track if the range is trivial. In particular, don't do the overlapping - // ranges check. - if ctor_range.is_singleton() { - split_ctors.push(IntRange(ctor_range)); - continue; - } - - /// Represents a border between 2 integers. Because the intervals spanning borders - /// must be able to cover every integer, we need to be able to represent - /// 2^128 + 1 such borders. - #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] - enum Border { - JustBefore(u128), - AfterMax, - } - - // A function for extracting the borders of an integer interval. - fn range_borders(r: IntRange<'_>) -> impl Iterator<Item = Border> { - let (lo, hi) = r.range.into_inner(); - let from = Border::JustBefore(lo); - let to = match hi.checked_add(1) { - Some(m) => Border::JustBefore(m), - None => Border::AfterMax, - }; - vec![from, to].into_iter() - } - - // Collect the span and range of all the intersecting ranges to lint on likely - // incorrect range patterns. (#63987) - let mut overlaps = vec![]; - // `borders` is the set of borders between equivalence classes: each equivalence - // class lies between 2 borders. - let row_borders = matrix - .patterns - .iter() - .flat_map(|row| { - IntRange::from_pat(tcx, param_env, row.head()).map(|r| (r, row.len())) - }) - .flat_map(|(range, row_len)| { - let intersection = ctor_range.intersection(tcx, &range); - let should_lint = ctor_range.suspicious_intersection(&range); - if let (Some(range), 1, true) = (&intersection, row_len, should_lint) { - // FIXME: for now, only check for overlapping ranges on simple range - // patterns. Otherwise with the current logic the following is detected - // as overlapping: - // match (10u8, true) { - // (0 ..= 125, false) => {} - // (126 ..= 255, false) => {} - // (0 ..= 255, true) => {} - // } - overlaps.push(range.clone()); - } - intersection - }) - .flat_map(range_borders); - let ctor_borders = range_borders(ctor_range.clone()); - let mut borders: Vec<_> = row_borders.chain(ctor_borders).collect(); - borders.sort_unstable(); - - lint_overlapping_patterns(tcx, hir_id, ctor_range, ty, overlaps); - - // We're going to iterate through every adjacent pair of borders, making sure that - // each represents an interval of nonnegative length, and convert each such - // interval into a constructor. - split_ctors.extend( - borders - .array_windows() - .filter_map(|&pair| match pair { - [Border::JustBefore(n), Border::JustBefore(m)] => { - if n < m { - Some(IntRange { range: n..=(m - 1), ty, span }) - } else { - None - } - } - [Border::JustBefore(n), Border::AfterMax] => { - Some(IntRange { range: n..=u128::MAX, ty, span }) - } - [Border::AfterMax, _] => None, - }) - .map(IntRange), - ); - } - Slice(Slice { array_len, kind: VarLen(self_prefix, self_suffix) }) => { - // The exhaustiveness-checking paper does not include any details on - // checking variable-length slice patterns. However, they are matched - // by an infinite collection of fixed-length array patterns. - // - // Checking the infinite set directly would take an infinite amount - // of time. However, it turns out that for each finite set of - // patterns `P`, all sufficiently large array lengths are equivalent: - // - // Each slice `s` with a "sufficiently-large" length `l ≥ L` that applies - // to exactly the subset `Pâ‚œ` of `P` can be transformed to a slice - // `sₘ` for each sufficiently-large length `m` that applies to exactly - // the same subset of `P`. - // - // Because of that, each witness for reachability-checking from one - // of the sufficiently-large lengths can be transformed to an - // equally-valid witness from any other length, so we only have - // to check slice lengths from the "minimal sufficiently-large length" - // and below. - // - // Note that the fact that there is a *single* `sₘ` for each `m` - // not depending on the specific pattern in `P` is important: if - // you look at the pair of patterns - // `[true, ..]` - // `[.., false]` - // Then any slice of length ≥1 that matches one of these two - // patterns can be trivially turned to a slice of any - // other length ≥1 that matches them and vice-versa - for - // but the slice from length 2 `[false, true]` that matches neither - // of these patterns can't be turned to a slice from length 1 that - // matches neither of these patterns, so we have to consider - // slices from length 2 there. - // - // Now, to see that that length exists and find it, observe that slice - // patterns are either "fixed-length" patterns (`[_, _, _]`) or - // "variable-length" patterns (`[_, .., _]`). - // - // For fixed-length patterns, all slices with lengths *longer* than - // the pattern's length have the same outcome (of not matching), so - // as long as `L` is greater than the pattern's length we can pick - // any `sₘ` from that length and get the same result. - // - // For variable-length patterns, the situation is more complicated, - // because as seen above the precise value of `sₘ` matters. - // - // However, for each variable-length pattern `p` with a prefix of length - // `plâ‚š` and suffix of length `slâ‚š`, only the first `plâ‚š` and the last - // `slâ‚š` elements are examined. - // - // Therefore, as long as `L` is positive (to avoid concerns about empty - // types), all elements after the maximum prefix length and before - // the maximum suffix length are not examined by any variable-length - // pattern, and therefore can be added/removed without affecting - // them - creating equivalent patterns from any sufficiently-large - // length. - // - // Of course, if fixed-length patterns exist, we must be sure - // that our length is large enough to miss them all, so - // we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))` - // - // for example, with the above pair of patterns, all elements - // but the first and last can be added/removed, so any - // witness of length ≥2 (say, `[false, false, true]`) can be - // turned to a witness from any other length ≥2. - - let mut max_prefix_len = self_prefix; - let mut max_suffix_len = self_suffix; - let mut max_fixed_len = 0; - - let head_ctors = - matrix.heads().filter_map(|pat| pat_constructor(tcx, param_env, pat)); - for ctor in head_ctors { - if let Slice(slice) = ctor { - match slice.pattern_kind() { - FixedLen(len) => { - max_fixed_len = cmp::max(max_fixed_len, len); - } - VarLen(prefix, suffix) => { - max_prefix_len = cmp::max(max_prefix_len, prefix); - max_suffix_len = cmp::max(max_suffix_len, suffix); - } - } - } - } - - // For diagnostics, we keep the prefix and suffix lengths separate, so in the case - // where `max_fixed_len + 1` is the largest, we adapt `max_prefix_len` accordingly, - // so that `L = max_prefix_len + max_suffix_len`. - if max_fixed_len + 1 >= max_prefix_len + max_suffix_len { - // The subtraction can't overflow thanks to the above check. - // The new `max_prefix_len` is also guaranteed to be larger than its previous - // value. - max_prefix_len = max_fixed_len + 1 - max_suffix_len; - } - - match array_len { - Some(len) => { - let kind = if max_prefix_len + max_suffix_len < len { - VarLen(max_prefix_len, max_suffix_len) - } else { - FixedLen(len) - }; - split_ctors.push(Slice(Slice { array_len, kind })); - } - None => { - // `ctor` originally covered the range `(self_prefix + - // self_suffix..infinity)`. We now split it into two: lengths smaller than - // `max_prefix_len + max_suffix_len` are treated independently as - // fixed-lengths slices, and lengths above are captured by a final VarLen - // constructor. - split_ctors.extend( - (self_prefix + self_suffix..max_prefix_len + max_suffix_len) - .map(|len| Slice(Slice { array_len, kind: FixedLen(len) })), - ); - split_ctors.push(Slice(Slice { - array_len, - kind: VarLen(max_prefix_len, max_suffix_len), - })); - } - } - } - // Any other constructor can be used unchanged. - _ => split_ctors.push(ctor), - } - } - - debug!("split_grouped_constructors(..)={:#?}", split_ctors); - split_ctors -} - -fn lint_overlapping_patterns<'tcx>( - tcx: TyCtxt<'tcx>, - hir_id: Option<HirId>, - ctor_range: IntRange<'tcx>, - ty: Ty<'tcx>, - overlaps: Vec<IntRange<'tcx>>, -) { - if let (true, Some(hir_id)) = (!overlaps.is_empty(), hir_id) { - tcx.struct_span_lint_hir( - lint::builtin::OVERLAPPING_PATTERNS, - hir_id, - ctor_range.span, - |lint| { - let mut err = lint.build("multiple patterns covering the same range"); - err.span_label(ctor_range.span, "overlapping patterns"); - for int_range in overlaps { - // Use the real type for user display of the ranges: - err.span_label( - int_range.span, - &format!( - "this range overlaps on `{}`", - IntRange { range: int_range.range, ty, span: DUMMY_SP }.to_pat(tcx), - ), - ); - } - err.emit(); - }, - ); - } -} - -fn constructor_covered_by_range<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ctor: &Constructor<'tcx>, - pat: &Pat<'tcx>, -) -> Option<()> { - if let Single = ctor { - return Some(()); - } - - let (pat_from, pat_to, pat_end, ty) = match *pat.kind { - PatKind::Constant { value } => (value, value, RangeEnd::Included, value.ty), - PatKind::Range(PatRange { lo, hi, end }) => (lo, hi, end, lo.ty), - _ => bug!("`constructor_covered_by_range` called with {:?}", pat), - }; - let (ctor_from, ctor_to, ctor_end) = match *ctor { - ConstantValue(value) => (value, value, RangeEnd::Included), - FloatRange(from, to, ctor_end) => (from, to, ctor_end), - _ => bug!("`constructor_covered_by_range` called with {:?}", ctor), - }; - trace!("constructor_covered_by_range {:#?}, {:#?}, {:#?}, {}", ctor, pat_from, pat_to, ty); - - let to = compare_const_vals(tcx, ctor_to, pat_to, param_env, ty)?; - let from = compare_const_vals(tcx, ctor_from, pat_from, param_env, ty)?; - let intersects = (from == Ordering::Greater || from == Ordering::Equal) - && (to == Ordering::Less || (pat_end == ctor_end && to == Ordering::Equal)); - if intersects { Some(()) } else { None } -} - -/// This is the main specialization step. It expands the pattern -/// into `arity` patterns based on the constructor. For most patterns, the step is trivial, -/// for instance tuple patterns are flattened and box patterns expand into their inner pattern. -/// Returns `None` if the pattern does not have the given constructor. -/// -/// OTOH, slice patterns with a subslice pattern (tail @ ..) can be expanded into multiple -/// different patterns. -/// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing -/// fields filled with wild patterns. -/// -/// This is roughly the inverse of `Constructor::apply`. -fn specialize_one_pattern<'p, 'tcx>( - cx: &mut MatchCheckCtxt<'p, 'tcx>, - pat: &'p Pat<'tcx>, - constructor: &Constructor<'tcx>, - ctor_wild_subpatterns: &Fields<'p, 'tcx>, -) -> Option<Fields<'p, 'tcx>> { - if let NonExhaustive = constructor { - // Only a wildcard pattern can match the special extra constructor - if !pat.is_wildcard() { - return None; - } - return Some(Fields::empty()); - } - - let result = match *pat.kind { - PatKind::AscribeUserType { .. } => bug!(), // Handled by `expand_pattern` - - PatKind::Binding { .. } | PatKind::Wild => Some(ctor_wild_subpatterns.clone()), - - PatKind::Variant { adt_def, variant_index, ref subpatterns, .. } => { - let variant = &adt_def.variants[variant_index]; - if constructor != &Variant(variant.def_id) { - return None; - } - Some(ctor_wild_subpatterns.replace_with_fieldpats(subpatterns)) - } - - PatKind::Leaf { ref subpatterns } => { - Some(ctor_wild_subpatterns.replace_with_fieldpats(subpatterns)) - } - - PatKind::Deref { ref subpattern } => Some(Fields::from_single_pattern(subpattern)), - - PatKind::Constant { value } if constructor.is_slice() => { - // We extract an `Option` for the pointer because slices of zero - // elements don't necessarily point to memory, they are usually - // just integers. The only time they should be pointing to memory - // is when they are subslices of nonzero slices. - let (alloc, offset, n, ty) = match value.ty.kind() { - ty::Array(t, n) => { - let n = n.eval_usize(cx.tcx, cx.param_env); - // Shortcut for `n == 0` where no matter what `alloc` and `offset` we produce, - // the result would be exactly what we early return here. - if n == 0 { - if ctor_wild_subpatterns.len() as u64 != n { - return None; - } - return Some(Fields::empty()); - } - match value.val { - ty::ConstKind::Value(ConstValue::ByRef { offset, alloc, .. }) => { - (Cow::Borrowed(alloc), offset, n, t) - } - _ => span_bug!(pat.span, "array pattern is {:?}", value,), - } - } - ty::Slice(t) => { - match value.val { - ty::ConstKind::Value(ConstValue::Slice { data, start, end }) => { - let offset = Size::from_bytes(start); - let n = (end - start) as u64; - (Cow::Borrowed(data), offset, n, t) - } - ty::ConstKind::Value(ConstValue::ByRef { .. }) => { - // FIXME(oli-obk): implement `deref` for `ConstValue` - return None; - } - _ => span_bug!( - pat.span, - "slice pattern constant must be scalar pair but is {:?}", - value, - ), - } - } - _ => span_bug!( - pat.span, - "unexpected const-val {:?} with ctor {:?}", - value, - constructor, - ), - }; - if ctor_wild_subpatterns.len() as u64 != n { - return None; - } - - // Convert a constant slice/array pattern to a list of patterns. - let layout = cx.tcx.layout_of(cx.param_env.and(ty)).ok()?; - let ptr = Pointer::new(AllocId(0), offset); - let pats = cx.pattern_arena.alloc_from_iter((0..n).filter_map(|i| { - let ptr = ptr.offset(layout.size * i, &cx.tcx).ok()?; - let scalar = alloc.read_scalar(&cx.tcx, ptr, layout.size).ok()?; - let scalar = scalar.check_init().ok()?; - let value = ty::Const::from_scalar(cx.tcx, scalar, ty); - let pattern = Pat { ty, span: pat.span, kind: box PatKind::Constant { value } }; - Some(pattern) - })); - // Ensure none of the dereferences failed. - if pats.len() as u64 != n { - return None; - } - Some(Fields::from_slice_unfiltered(pats)) - } - - PatKind::Constant { .. } | PatKind::Range { .. } => { - // If the constructor is a: - // - Single value: add a row if the pattern contains the constructor. - // - Range: add a row if the constructor intersects the pattern. - if let IntRange(ctor) = constructor { - let pat = IntRange::from_pat(cx.tcx, cx.param_env, pat)?; - ctor.intersection(cx.tcx, &pat)?; - // Constructor splitting should ensure that all intersections we encounter - // are actually inclusions. - assert!(ctor.is_subrange(&pat)); - } else { - // Fallback for non-ranges and ranges that involve - // floating-point numbers, which are not conveniently handled - // by `IntRange`. For these cases, the constructor may not be a - // range so intersection actually devolves into being covered - // by the pattern. - constructor_covered_by_range(cx.tcx, cx.param_env, constructor, pat)?; - } - Some(Fields::empty()) - } - - PatKind::Array { ref prefix, ref slice, ref suffix } - | PatKind::Slice { ref prefix, ref slice, ref suffix } => match *constructor { - Slice(_) => { - // Number of subpatterns for this pattern - let pat_len = prefix.len() + suffix.len(); - // Number of subpatterns for this constructor - let arity = ctor_wild_subpatterns.len(); - - if (slice.is_none() && arity != pat_len) || pat_len > arity { - return None; - } - - // Replace the prefix and the suffix with the given patterns, leaving wildcards in - // the middle if there was a subslice pattern `..`. - let prefix = prefix.iter().enumerate(); - let suffix = suffix.iter().enumerate().map(|(i, p)| (arity - suffix.len() + i, p)); - Some(ctor_wild_subpatterns.replace_fields_indexed(prefix.chain(suffix))) - } - ConstantValue(cv) => { - match slice_pat_covered_by_const( - cx.tcx, - pat.span, - cv, - prefix, - slice, - suffix, - cx.param_env, - ) { - Ok(true) => Some(Fields::empty()), - Ok(false) => None, - Err(ErrorReported) => None, - } - } - _ => span_bug!(pat.span, "unexpected ctor {:?} for slice pat", constructor), - }, - - PatKind::Or { .. } => bug!("Or-pattern should have been expanded earlier on."), - }; - debug!( - "specialize({:#?}, {:#?}, {:#?}) = {:#?}", - pat, constructor, ctor_wild_subpatterns, result - ); - - result -} diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs index 69de7c7e2ee..30b700a1d4f 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs @@ -137,7 +137,7 @@ impl<'tcx> MatchVisitor<'_, 'tcx> { patcx.include_lint_checks(); let pattern = patcx.lower_pattern(pat); let pattern_ty = pattern.ty; - let pattern: &_ = cx.pattern_arena.alloc(expand_pattern(cx, pattern)); + let pattern: &_ = cx.pattern_arena.alloc(expand_pattern(pattern)); if !patcx.errors.is_empty() { *have_errors = true; patcx.report_inlining_errors(pat.span); diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs index a203b3a1428..6370f8c375b 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs @@ -387,14 +387,16 @@ impl<'a, 'tcx> ConstToPat<'a, 'tcx> { // `&str` and `&[u8]` are represented as `ConstValue::Slice`, let's keep using this // optimization for now. ty::Str => PatKind::Constant { value: cv }, - ty::Slice(elem_ty) if elem_ty == tcx.types.u8 => PatKind::Constant { value: cv }, // `b"foo"` produces a `&[u8; 3]`, but you can't use constants of array type when // matching against references, you can only use byte string literals. - // FIXME: clean this up, likely by permitting array patterns when matching on slices - ty::Array(elem_ty, _) if elem_ty == tcx.types.u8 => PatKind::Constant { value: cv }, + // The typechecker has a special case for byte string literals, by treating them + // as slices. This means we turn `&[T; N]` constants into slice patterns, which + // has no negative effects on pattern matching, even if we're actually matching on + // arrays. + ty::Array(..) | // Cannot merge this with the catch all branch below, because the `const_deref` - // changes the type from slice to array, and slice patterns behave differently from - // array patterns. + // changes the type from slice to array, we need to keep the original type in the + // pattern. ty::Slice(..) => { let old = self.behind_reference.replace(true); let array = tcx.deref_const(self.param_env.and(cv)); diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs index b05ddb3b464..3c746ac5153 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs @@ -158,6 +158,13 @@ crate enum PatKind<'tcx> { subpattern: Pat<'tcx>, }, + /// One of the following: + /// * `&str`, which will be handled as a string pattern and thus exhaustiveness + /// checking will detect if you use the same string twice in different patterns. + /// * integer, bool, char or float, which will be handled by exhaustivenes to cover exactly + /// its own value, similar to `&str`, but these values are much simpler. + /// * Opaque constants, that must not be matched structurally. So anything that does not derive + /// `PartialEq` and `Eq`. Constant { value: &'tcx ty::Const<'tcx>, }, @@ -216,10 +223,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> { BindingMode::ByValue => mutability == Mutability::Mut, BindingMode::ByRef(bk) => { write!(f, "ref ")?; - match bk { - BorrowKind::Mut { .. } => true, - _ => false, - } + matches!(bk, BorrowKind::Mut { .. }) } }; if is_mut { diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 25deb46e147..e851451269e 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -8,7 +8,7 @@ use rustc_ast as ast; use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; -use rustc_ast::tokenstream::{self, TokenStream, TokenTree}; +use rustc_ast::tokenstream::{self, LazyTokenStream, TokenStream, TokenTree}; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; use rustc_errors::{Diagnostic, FatalError, Level, PResult}; @@ -22,7 +22,7 @@ use std::str; use tracing::{debug, info}; -pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); +pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); #[macro_use] pub mod parser; @@ -248,35 +248,37 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke // As a result, some AST nodes are annotated with the token stream they // came from. Here we attempt to extract these lossless token streams // before we fall back to the stringification. + + let convert_tokens = + |tokens: &Option<LazyTokenStream>| tokens.as_ref().map(|t| t.create_token_stream()); + let tokens = match *nt { - Nonterminal::NtItem(ref item) => { - prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) - } - Nonterminal::NtBlock(ref block) => block.tokens.clone(), + Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()), + Nonterminal::NtBlock(ref block) => convert_tokens(&block.tokens), Nonterminal::NtStmt(ref stmt) => { // FIXME: We currently only collect tokens for `:stmt` // matchers in `macro_rules!` macros. When we start collecting // tokens for attributes on statements, we will need to prepend // attributes here - stmt.tokens.clone() + convert_tokens(&stmt.tokens) } - Nonterminal::NtPat(ref pat) => pat.tokens.clone(), - Nonterminal::NtTy(ref ty) => ty.tokens.clone(), + Nonterminal::NtPat(ref pat) => convert_tokens(&pat.tokens), + Nonterminal::NtTy(ref ty) => convert_tokens(&ty.tokens), Nonterminal::NtIdent(ident, is_raw) => { Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) } Nonterminal::NtLifetime(ident) => { Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) } - Nonterminal::NtMeta(ref attr) => attr.tokens.clone(), - Nonterminal::NtPath(ref path) => path.tokens.clone(), - Nonterminal::NtVis(ref vis) => vis.tokens.clone(), + Nonterminal::NtMeta(ref attr) => convert_tokens(&attr.tokens), + Nonterminal::NtPath(ref path) => convert_tokens(&path.tokens), + Nonterminal::NtVis(ref vis) => convert_tokens(&vis.tokens), Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { if expr.tokens.is_none() { debug!("missing tokens for expr {:?}", expr); } - prepend_attrs(sess, &expr.attrs, expr.tokens.as_ref(), span) + prepend_attrs(&expr.attrs, expr.tokens.as_ref()) } }; @@ -600,14 +602,12 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { } fn prepend_attrs( - sess: &ParseSess, attrs: &[ast::Attribute], - tokens: Option<&tokenstream::TokenStream>, - span: rustc_span::Span, + tokens: Option<&tokenstream::LazyTokenStream>, ) -> Option<tokenstream::TokenStream> { - let tokens = tokens?; + let tokens = tokens?.create_token_stream(); if attrs.is_empty() { - return Some(tokens.clone()); + return Some(tokens); } let mut builder = tokenstream::TokenStreamBuilder::new(); for attr in attrs { @@ -616,48 +616,13 @@ fn prepend_attrs( ast::AttrStyle::Outer, "inner attributes should prevent cached tokens from existing" ); - - let source = pprust::attribute_to_string(attr); - let macro_filename = FileName::macro_expansion_source_code(&source); - - let item = match attr.kind { - ast::AttrKind::Normal(ref item) => item, - ast::AttrKind::DocComment(..) => { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - builder.push(stream); - continue; - } - }; - - // synthesize # [ $path $tokens ] manually here - let mut brackets = tokenstream::TokenStreamBuilder::new(); - - // For simple paths, push the identifier directly - if item.path.segments.len() == 1 && item.path.segments[0].args.is_none() { - let ident = item.path.segments[0].ident; - let token = token::Ident(ident.name, ident.as_str().starts_with("r#")); - brackets.push(tokenstream::TokenTree::token(token, ident.span)); - - // ... and for more complicated paths, fall back to a reparse hack that - // should eventually be removed. - } else { - let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); - brackets.push(stream); - } - - brackets.push(item.args.outer_tokens()); - - // The span we list here for `#` and for `[ ... ]` are both wrong in - // that it encompasses more than each token, but it hopefully is "good - // enough" for now at least. - builder.push(tokenstream::TokenTree::token(token::Pound, attr.span)); - let delim_span = tokenstream::DelimSpan::from_single(attr.span); - builder.push(tokenstream::TokenTree::Delimited( - delim_span, - token::DelimToken::Bracket, - brackets.build(), - )); + builder.push( + attr.tokens + .as_ref() + .unwrap_or_else(|| panic!("Attribute {:?} is missing tokens!", attr)) + .create_token_stream(), + ); } - builder.push(tokens.clone()); + builder.push(tokens); Some(builder.build()) } diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 98f94098bfc..053b7e0b75f 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -4,7 +4,7 @@ use rustc_ast::attr; use rustc_ast::token::{self, Nonterminal}; use rustc_ast_pretty::pprust; use rustc_errors::{error_code, PResult}; -use rustc_span::Span; +use rustc_span::{sym, Span}; use tracing::debug; @@ -30,41 +30,53 @@ impl<'a> Parser<'a> { let mut just_parsed_doc_comment = false; loop { debug!("parse_outer_attributes: self.token={:?}", self.token); - if self.check(&token::Pound) { - let inner_error_reason = if just_parsed_doc_comment { - "an inner attribute is not permitted following an outer doc comment" - } else if !attrs.is_empty() { - "an inner attribute is not permitted following an outer attribute" - } else { - DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG - }; - let inner_parse_policy = InnerAttrPolicy::Forbidden { - reason: inner_error_reason, - saw_doc_comment: just_parsed_doc_comment, - prev_attr_sp: attrs.last().map(|a| a.span), - }; - let attr = self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; - attrs.push(attr); - just_parsed_doc_comment = false; + let (attr, tokens) = if self.check(&token::Pound) { + self.collect_tokens(|this| { + let inner_error_reason = if just_parsed_doc_comment { + "an inner attribute is not permitted following an outer doc comment" + } else if !attrs.is_empty() { + "an inner attribute is not permitted following an outer attribute" + } else { + DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG + }; + let inner_parse_policy = InnerAttrPolicy::Forbidden { + reason: inner_error_reason, + saw_doc_comment: just_parsed_doc_comment, + prev_attr_sp: attrs.last().map(|a| a.span), + }; + let attr = this.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; + just_parsed_doc_comment = false; + Ok(Some(attr)) + })? } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { - let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); - if attr.style != ast::AttrStyle::Outer { - self.sess - .span_diagnostic - .struct_span_err_with_code( - self.token.span, - "expected outer doc comment", - error_code!(E0753), - ) - .note( - "inner doc comments like this (starting with \ - `//!` or `/*!`) can only appear before items", - ) - .emit(); - } + self.collect_tokens(|this| { + let attr = + attr::mk_doc_comment(comment_kind, attr_style, data, this.token.span); + if attr.style != ast::AttrStyle::Outer { + this.sess + .span_diagnostic + .struct_span_err_with_code( + this.token.span, + "expected outer doc comment", + error_code!(E0753), + ) + .note( + "inner doc comments like this (starting with \ + `//!` or `/*!`) can only appear before items", + ) + .emit(); + } + this.bump(); + just_parsed_doc_comment = true; + Ok(Some(attr)) + })? + } else { + (None, None) + }; + + if let Some(mut attr) = attr { + attr.tokens = tokens; attrs.push(attr); - self.bump(); - just_parsed_doc_comment = true; } else { break; } @@ -99,7 +111,7 @@ impl<'a> Parser<'a> { if self.eat(&token::Not) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer }; self.expect(&token::OpenDelim(token::Bracket))?; - let item = self.parse_attr_item()?; + let item = self.parse_attr_item(false)?; self.expect(&token::CloseDelim(token::Bracket))?; let attr_sp = lo.to(self.prev_token.span); @@ -148,7 +160,7 @@ impl<'a> Parser<'a> { /// PATH /// PATH `=` UNSUFFIXED_LIT /// The delimiters or `=` are still put into the resulting token stream. - pub fn parse_attr_item(&mut self) -> PResult<'a, ast::AttrItem> { + pub fn parse_attr_item(&mut self, capture_tokens: bool) -> PResult<'a, ast::AttrItem> { let item = match self.token.kind { token::Interpolated(ref nt) => match **nt { Nonterminal::NtMeta(ref item) => Some(item.clone().into_inner()), @@ -160,9 +172,18 @@ impl<'a> Parser<'a> { self.bump(); item } else { - let path = self.parse_path(PathStyle::Mod)?; - let args = self.parse_attr_args()?; - ast::AttrItem { path, args, tokens: None } + let do_parse = |this: &mut Self| { + let path = this.parse_path(PathStyle::Mod)?; + let args = this.parse_attr_args()?; + Ok(ast::AttrItem { path, args, tokens: None }) + }; + if capture_tokens { + let (mut item, tokens) = self.collect_tokens(do_parse)?; + item.tokens = tokens; + item + } else { + do_parse(self)? + } }) } @@ -175,19 +196,31 @@ impl<'a> Parser<'a> { let mut attrs: Vec<ast::Attribute> = vec![]; loop { // Only try to parse if it is an inner attribute (has `!`). - if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { - let attr = self.parse_attribute(true)?; - assert_eq!(attr.style, ast::AttrStyle::Inner); - attrs.push(attr); - } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { - // We need to get the position of this token before we bump. - let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); - if attr.style == ast::AttrStyle::Inner { - attrs.push(attr); - self.bump(); + let (attr, tokens) = + if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { + self.collect_tokens(|this| { + let attr = this.parse_attribute(true)?; + assert_eq!(attr.style, ast::AttrStyle::Inner); + Ok(Some(attr)) + })? + } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { + self.collect_tokens(|this| { + // We need to get the position of this token before we bump. + let attr = + attr::mk_doc_comment(comment_kind, attr_style, data, this.token.span); + if attr.style == ast::AttrStyle::Inner { + this.bump(); + Ok(Some(attr)) + } else { + Ok(None) + } + })? } else { - break; - } + (None, None) + }; + if let Some(mut attr) = attr { + attr.tokens = tokens; + attrs.push(attr); } else { break; } @@ -220,7 +253,7 @@ impl<'a> Parser<'a> { let mut expanded_attrs = Vec::with_capacity(1); while self.token.kind != token::Eof { let lo = self.token.span; - let item = self.parse_attr_item()?; + let item = self.parse_attr_item(true)?; expanded_attrs.push((item, lo.to(self.prev_token.span))); if !self.eat(&token::Comma) { break; @@ -302,3 +335,16 @@ impl<'a> Parser<'a> { Err(self.struct_span_err(self.token.span, &msg)) } } + +pub fn maybe_needs_tokens(attrs: &[ast::Attribute]) -> bool { + attrs.iter().any(|attr| { + if let Some(ident) = attr.ident() { + ident.name == sym::derive + // This might apply a custom attribute/derive + || ident.name == sym::cfg_attr + || !rustc_feature::is_builtin_attr_name(ident.name) + } else { + true + } + }) +} diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 1ea01d95a13..cd3b8db2303 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -1,13 +1,14 @@ use super::ty::AllowPlus; -use super::{BlockMode, Parser, PathStyle, SemiColonMode, SeqSep, TokenExpectType, TokenType}; +use super::TokenType; +use super::{BlockMode, Parser, PathStyle, Restrictions, SemiColonMode, SeqSep, TokenExpectType}; use rustc_ast::ptr::P; use rustc_ast::token::{self, Lit, LitKind, TokenKind}; use rustc_ast::util::parser::AssocOp; use rustc_ast::{ - self as ast, AngleBracketedArgs, AttrVec, BinOpKind, BindingMode, Block, BlockCheckMode, Expr, - ExprKind, Item, ItemKind, Mutability, Param, Pat, PatKind, Path, PathSegment, QSelf, Ty, - TyKind, + self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AttrVec, BinOpKind, BindingMode, + Block, BlockCheckMode, Expr, ExprKind, GenericArg, Item, ItemKind, Mutability, Param, Pat, + PatKind, Path, PathSegment, QSelf, Ty, TyKind, }; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashSet; @@ -19,7 +20,8 @@ use rustc_span::{MultiSpan, Span, SpanSnippetError, DUMMY_SP}; use tracing::{debug, trace}; -const TURBOFISH: &str = "use `::<...>` instead of `<...>` to specify type arguments"; +const TURBOFISH_SUGGESTION_STR: &str = + "use `::<...>` instead of `<...>` to specify type or const arguments"; /// Creates a placeholder argument. pub(super) fn dummy_arg(ident: Ident) -> Param { @@ -658,7 +660,7 @@ impl<'a> Parser<'a> { Ok(_) => { e.span_suggestion_verbose( binop.span.shrink_to_lo(), - "use `::<...>` instead of `<...>` to specify type arguments", + TURBOFISH_SUGGESTION_STR, "::".to_string(), Applicability::MaybeIncorrect, ); @@ -813,7 +815,7 @@ impl<'a> Parser<'a> { let suggest = |err: &mut DiagnosticBuilder<'_>| { err.span_suggestion_verbose( op.span.shrink_to_lo(), - TURBOFISH, + TURBOFISH_SUGGESTION_STR, "::".to_string(), Applicability::MaybeIncorrect, ); @@ -887,7 +889,7 @@ impl<'a> Parser<'a> { { // All we know is that this is `foo < bar >` and *nothing* else. Try to // be helpful, but don't attempt to recover. - err.help(TURBOFISH); + err.help(TURBOFISH_SUGGESTION_STR); err.help("or use `(...)` if you meant to specify fn arguments"); } @@ -1207,7 +1209,13 @@ impl<'a> Parser<'a> { self.recover_await_prefix(await_sp)? }; let sp = self.error_on_incorrect_await(lo, hi, &expr, is_question); - let expr = self.mk_expr(lo.to(sp), ExprKind::Await(expr), attrs); + let kind = match expr.kind { + // Avoid knock-down errors as we don't know whether to interpret this as `foo().await?` + // or `foo()?.await` (the very reason we went with postfix syntax 😅). + ExprKind::Try(_) => ExprKind::Err, + _ => ExprKind::Await(expr), + }; + let expr = self.mk_expr(lo.to(sp), kind, attrs); self.maybe_recover_from_bad_qpath(expr, true) } @@ -1351,11 +1359,7 @@ impl<'a> Parser<'a> { (self.token == token::Lt && // `foo:<bar`, likely a typoed turbofish. self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())) || self.token.is_ident() && - match node { - // `foo::` → `foo:` or `foo.bar::` → `foo.bar:` - ast::ExprKind::Path(..) | ast::ExprKind::Field(..) => true, - _ => false, - } && + matches!(node, ast::ExprKind::Path(..) | ast::ExprKind::Field(..)) && !self.token.is_reserved_ident() && // v `foo:bar(baz)` self.look_ahead(1, |t| t == &token::OpenDelim(token::Paren)) || self.look_ahead(1, |t| t == &token::OpenDelim(token::Brace)) // `foo:bar {` @@ -1550,14 +1554,6 @@ impl<'a> Parser<'a> { } } - pub(super) fn expected_semi_or_open_brace<T>(&mut self) -> PResult<'a, T> { - let token_str = super::token_descr(&self.token); - let msg = &format!("expected `;` or `{{`, found {}", token_str); - let mut err = self.struct_span_err(self.token.span, msg); - err.span_label(self.token.span, "expected `;` or `{`"); - Err(err) - } - pub(super) fn eat_incorrect_doc_comment_for_param_type(&mut self) { if let token::DocComment(..) = self.token.kind { self.struct_span_err( @@ -1774,4 +1770,142 @@ impl<'a> Parser<'a> { } } } + + /// Handle encountering a symbol in a generic argument list that is not a `,` or `>`. In this + /// case, we emit an error and try to suggest enclosing a const argument in braces if it looks + /// like the user has forgotten them. + pub fn handle_ambiguous_unbraced_const_arg( + &mut self, + args: &mut Vec<AngleBracketedArg>, + ) -> PResult<'a, bool> { + // If we haven't encountered a closing `>`, then the argument is malformed. + // It's likely that the user has written a const expression without enclosing it + // in braces, so we try to recover here. + let arg = args.pop().unwrap(); + // FIXME: for some reason using `unexpected` or `expected_one_of_not_found` has + // adverse side-effects to subsequent errors and seems to advance the parser. + // We are causing this error here exclusively in case that a `const` expression + // could be recovered from the current parser state, even if followed by more + // arguments after a comma. + let mut err = self.struct_span_err( + self.token.span, + &format!("expected one of `,` or `>`, found {}", super::token_descr(&self.token)), + ); + err.span_label(self.token.span, "expected one of `,` or `>`"); + match self.recover_const_arg(arg.span(), err) { + Ok(arg) => { + args.push(AngleBracketedArg::Arg(arg)); + if self.eat(&token::Comma) { + return Ok(true); // Continue + } + } + Err(mut err) => { + args.push(arg); + // We will emit a more generic error later. + err.delay_as_bug(); + } + } + return Ok(false); // Don't continue. + } + + /// Handle a generic const argument that had not been enclosed in braces, and suggest enclosing + /// it braces. In this situation, unlike in `handle_ambiguous_unbraced_const_arg`, this is + /// almost certainly a const argument, so we always offer a suggestion. + pub fn handle_unambiguous_unbraced_const_arg(&mut self) -> PResult<'a, P<Expr>> { + let start = self.token.span; + let expr = self.parse_expr_res(Restrictions::CONST_EXPR, None).map_err(|mut err| { + err.span_label( + start.shrink_to_lo(), + "while parsing a const generic argument starting here", + ); + err + })?; + if !self.expr_is_valid_const_arg(&expr) { + self.struct_span_err( + expr.span, + "expressions must be enclosed in braces to be used as const generic \ + arguments", + ) + .multipart_suggestion( + "enclose the `const` expression in braces", + vec![ + (expr.span.shrink_to_lo(), "{ ".to_string()), + (expr.span.shrink_to_hi(), " }".to_string()), + ], + Applicability::MachineApplicable, + ) + .emit(); + } + Ok(expr) + } + + /// Try to recover from possible generic const argument without `{` and `}`. + /// + /// When encountering code like `foo::< bar + 3 >` or `foo::< bar - baz >` we suggest + /// `foo::<{ bar + 3 }>` and `foo::<{ bar - baz }>`, respectively. We only provide a suggestion + /// if we think that that the resulting expression would be well formed. + pub fn recover_const_arg( + &mut self, + start: Span, + mut err: DiagnosticBuilder<'a>, + ) -> PResult<'a, GenericArg> { + let is_op = AssocOp::from_token(&self.token) + .and_then(|op| { + if let AssocOp::Greater + | AssocOp::Less + | AssocOp::ShiftRight + | AssocOp::GreaterEqual + // Don't recover from `foo::<bar = baz>`, because this could be an attempt to + // assign a value to a defaulted generic parameter. + | AssocOp::Assign + | AssocOp::AssignOp(_) = op + { + None + } else { + Some(op) + } + }) + .is_some(); + // This will be true when a trait object type `Foo +` or a path which was a `const fn` with + // type params has been parsed. + let was_op = + matches!(self.prev_token.kind, token::BinOp(token::Plus | token::Shr) | token::Gt); + if !is_op && !was_op { + // We perform these checks and early return to avoid taking a snapshot unnecessarily. + return Err(err); + } + let snapshot = self.clone(); + if is_op { + self.bump(); + } + match self.parse_expr_res(Restrictions::CONST_EXPR, None) { + Ok(expr) => { + if token::Comma == self.token.kind || self.token.kind.should_end_const_arg() { + // Avoid the following output by checking that we consumed a full const arg: + // help: expressions must be enclosed in braces to be used as const generic + // arguments + // | + // LL | let sr: Vec<{ (u32, _, _) = vec![] }; + // | ^ ^ + err.multipart_suggestion( + "expressions must be enclosed in braces to be used as const generic \ + arguments", + vec![ + (start.shrink_to_lo(), "{ ".to_string()), + (expr.span.shrink_to_hi(), " }".to_string()), + ], + Applicability::MaybeIncorrect, + ); + let value = self.mk_expr_err(start.to(expr.span)); + err.emit(); + return Ok(GenericArg::Const(AnonConst { id: ast::DUMMY_NODE_ID, value })); + } + } + Err(mut err) => { + err.cancel(); + } + } + *self = snapshot; + Err(err) + } } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index fb05f8791a5..c2a13d4b0de 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -6,6 +6,7 @@ use crate::maybe_recover_from_interpolated_ty_qpath; use rustc_ast::ptr::P; use rustc_ast::token::{self, Token, TokenKind}; +use rustc_ast::tokenstream::Spacing; use rustc_ast::util::classify; use rustc_ast::util::literal::LitError; use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity}; @@ -18,7 +19,6 @@ use rustc_span::source_map::{self, Span, Spanned}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{BytePos, Pos}; use std::mem; -use tracing::debug; /// Possibly accepts an `token::Interpolated` expression (a pre-parsed expression /// dropped into the token stream, which happens while parsing the result of @@ -359,6 +359,18 @@ impl<'a> Parser<'a> { /// Also performs recovery for `and` / `or` which are mistaken for `&&` and `||` respectively. fn check_assoc_op(&self) -> Option<Spanned<AssocOp>> { let (op, span) = match (AssocOp::from_token(&self.token), self.token.ident()) { + // When parsing const expressions, stop parsing when encountering `>`. + ( + Some( + AssocOp::ShiftRight + | AssocOp::Greater + | AssocOp::GreaterEqual + | AssocOp::AssignOp(token::BinOpToken::Shr), + ), + _, + ) if self.restrictions.contains(Restrictions::CONST_EXPR) => { + return None; + } (Some(op), _) => (op, self.token.span), (None, Some((Ident { name: sym::and, span }, false))) => { self.error_bad_logical_op("and", "&&", "conjunction"); @@ -459,7 +471,7 @@ impl<'a> Parser<'a> { /// Parses a prefix-unary-operator expr. fn parse_prefix_expr(&mut self, attrs: Option<AttrVec>) -> PResult<'a, P<Expr>> { let attrs = self.parse_or_use_outer_attributes(attrs)?; - self.maybe_collect_tokens(!attrs.is_empty(), |this| { + self.maybe_collect_tokens(super::attr::maybe_needs_tokens(&attrs), |this| { let lo = this.token.span; // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() let (hi, ex) = match this.token.uninterpolate().kind { @@ -819,7 +831,7 @@ impl<'a> Parser<'a> { self.struct_span_err(self.token.span, &format!("unexpected token: `{}`", actual)).emit(); } - // We need and identifier or integer, but the next token is a float. + // We need an identifier or integer, but the next token is a float. // Break the float into components to extract the identifier or integer. // FIXME: With current `TokenCursor` it's hard to break tokens into more than 2 // parts unless those parts are processed immediately. `TokenCursor` should either @@ -884,7 +896,7 @@ impl<'a> Parser<'a> { assert!(suffix.is_none()); let symbol = Symbol::intern(&i); self.token = Token::new(token::Ident(symbol, false), ident_span); - let next_token = Token::new(token::Dot, dot_span); + let next_token = (Token::new(token::Dot, dot_span), self.token_spacing); self.parse_tuple_field_access_expr(lo, base, symbol, None, Some(next_token)) } // 1.2 | 1.2e3 @@ -902,12 +914,14 @@ impl<'a> Parser<'a> { }; let symbol1 = Symbol::intern(&i1); self.token = Token::new(token::Ident(symbol1, false), ident1_span); - let next_token1 = Token::new(token::Dot, dot_span); + // This needs to be `Spacing::Alone` to prevent regressions. + // See issue #76399 and PR #76285 for more details + let next_token1 = (Token::new(token::Dot, dot_span), Spacing::Alone); let base1 = self.parse_tuple_field_access_expr(lo, base, symbol1, None, Some(next_token1)); let symbol2 = Symbol::intern(&i2); let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span); - self.bump_with(next_token2); // `.` + self.bump_with((next_token2, self.token_spacing)); // `.` self.parse_tuple_field_access_expr(lo, base1, symbol2, suffix, None) } // 1e+ | 1e- (recovered) @@ -930,7 +944,7 @@ impl<'a> Parser<'a> { base: P<Expr>, field: Symbol, suffix: Option<Symbol>, - next_token: Option<Token>, + next_token: Option<(Token, Spacing)>, ) -> P<Expr> { match next_token { Some(next_token) => self.bump_with(next_token), @@ -1060,8 +1074,8 @@ impl<'a> Parser<'a> { }) } else if self.eat_keyword(kw::Unsafe) { self.parse_block_expr(None, lo, BlockCheckMode::Unsafe(ast::UserProvided), attrs) - } else if self.check_inline_const() { - self.parse_const_expr(lo.to(self.token.span)) + } else if self.check_inline_const(0) { + self.parse_const_block(lo.to(self.token.span)) } else if self.is_do_catch_block() { self.recover_do_catch(attrs) } else if self.is_try_block() { @@ -1109,13 +1123,12 @@ impl<'a> Parser<'a> { fn maybe_collect_tokens( &mut self, - has_outer_attrs: bool, + needs_tokens: bool, f: impl FnOnce(&mut Self) -> PResult<'a, P<Expr>>, ) -> PResult<'a, P<Expr>> { - if has_outer_attrs { + if needs_tokens { let (mut expr, tokens) = self.collect_tokens(f)?; - debug!("maybe_collect_tokens: Collected tokens for {:?} (tokens {:?}", expr, tokens); - expr.tokens = Some(tokens); + expr.tokens = tokens; Ok(expr) } else { f(self) @@ -1714,7 +1727,7 @@ impl<'a> Parser<'a> { let lo = self.prev_token.span; let pat = self.parse_top_pat(GateOr::No)?; self.expect(&token::Eq)?; - let expr = self.with_res(Restrictions::NO_STRUCT_LITERAL, |this| { + let expr = self.with_res(self.restrictions | Restrictions::NO_STRUCT_LITERAL, |this| { this.parse_assoc_expr_with(1 + prec_let_scrutinee_needs_par(), None.into()) })?; let span = lo.to(expr.span); diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 48341f71d33..48a635844fe 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -116,15 +116,16 @@ impl<'a> Parser<'a> { Some(item.into_inner()) }); + let needs_tokens = super::attr::maybe_needs_tokens(&attrs); + let mut unclosed_delims = vec![]; - let has_attrs = !attrs.is_empty(); let parse_item = |this: &mut Self| { let item = this.parse_item_common_(attrs, mac_allowed, attrs_allowed, req_name); unclosed_delims.append(&mut this.unclosed_delims); item }; - let (mut item, tokens) = if has_attrs { + let (mut item, tokens) = if needs_tokens { let (item, tokens) = self.collect_tokens(parse_item)?; (item, Some(tokens)) } else { @@ -150,7 +151,7 @@ impl<'a> Parser<'a> { if let Some(tokens) = tokens { if let Some(item) = &mut item { if !item.attrs.iter().any(|attr| attr.style == AttrStyle::Inner) { - item.tokens = Some(tokens); + item.tokens = tokens; } } } @@ -375,21 +376,19 @@ impl<'a> Parser<'a> { format!(" {} ", kw), Applicability::MachineApplicable, ); + } else if let Ok(snippet) = self.span_to_snippet(ident_sp) { + err.span_suggestion( + full_sp, + "if you meant to call a macro, try", + format!("{}!", snippet), + // this is the `ambiguous` conditional branch + Applicability::MaybeIncorrect, + ); } else { - if let Ok(snippet) = self.span_to_snippet(ident_sp) { - err.span_suggestion( - full_sp, - "if you meant to call a macro, try", - format!("{}!", snippet), - // this is the `ambiguous` conditional branch - Applicability::MaybeIncorrect, - ); - } else { - err.help( - "if you meant to call a macro, remove the `pub` \ - and add a trailing `!` after the identifier", - ); - } + err.help( + "if you meant to call a macro, remove the `pub` \ + and add a trailing `!` after the identifier", + ); } Err(err) } else if self.look_ahead(1, |t| *t == token::Lt) { @@ -981,10 +980,7 @@ impl<'a> Parser<'a> { if token.is_keyword(kw::Move) { return true; } - match token.kind { - token::BinOp(token::Or) | token::OrOr => true, - _ => false, - } + matches!(token.kind, token::BinOp(token::Or) | token::OrOr) }) } else { false @@ -1537,7 +1533,7 @@ impl<'a> Parser<'a> { generics.where_clause = self.parse_where_clause()?; // `where T: Ord` let mut sig_hi = self.prev_token.span; - let body = self.parse_fn_body(attrs, &mut sig_hi)?; // `;` or `{ ... }`. + let body = self.parse_fn_body(attrs, &ident, &mut sig_hi)?; // `;` or `{ ... }`. let fn_sig_span = sig_lo.to(sig_hi); Ok((ident, FnSig { header, decl, span: fn_sig_span }, generics, body)) } @@ -1548,12 +1544,12 @@ impl<'a> Parser<'a> { fn parse_fn_body( &mut self, attrs: &mut Vec<Attribute>, + ident: &Ident, sig_hi: &mut Span, ) -> PResult<'a, Option<P<Block>>> { - let (inner_attrs, body) = if self.check(&token::Semi) { + let (inner_attrs, body) = if self.eat(&token::Semi) { // Include the trailing semicolon in the span of the signature - *sig_hi = self.token.span; - self.bump(); // `;` + *sig_hi = self.prev_token.span; (Vec::new(), None) } else if self.check(&token::OpenDelim(token::Brace)) || self.token.is_whole_block() { self.parse_inner_attrs_and_block().map(|(attrs, body)| (attrs, Some(body)))? @@ -1573,7 +1569,21 @@ impl<'a> Parser<'a> { .emit(); (Vec::new(), Some(self.mk_block_err(span))) } else { - return self.expected_semi_or_open_brace(); + if let Err(mut err) = + self.expected_one_of_not_found(&[], &[token::Semi, token::OpenDelim(token::Brace)]) + { + if self.token.kind == token::CloseDelim(token::Brace) { + // The enclosing `mod`, `trait` or `impl` is being closed, so keep the `fn` in + // the AST for typechecking. + err.span_label(ident.span, "while parsing this `fn`"); + err.emit(); + (Vec::new(), None) + } else { + return Err(err); + } + } else { + unreachable!() + } }; attrs.extend(inner_attrs); Ok(body) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 7970ad36456..da1c54e88b5 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -16,13 +16,15 @@ pub use path::PathStyle; use rustc_ast::ptr::P; use rustc_ast::token::{self, DelimToken, Token, TokenKind}; -use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndSpacing}; +use rustc_ast::tokenstream::{self, DelimSpan, LazyTokenStream, Spacing}; +use rustc_ast::tokenstream::{CreateTokenStream, TokenStream, TokenTree}; use rustc_ast::DUMMY_NODE_ID; use rustc_ast::{self as ast, AnonConst, AttrStyle, AttrVec, Const, CrateSugar, Extern, Unsafe}; use rustc_ast::{Async, Expr, ExprKind, MacArgs, MacDelimiter, Mutability, StrLit}; use rustc_ast::{Visibility, VisibilityKind}; use rustc_ast_pretty::pprust; -use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult}; +use rustc_errors::PResult; +use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError}; use rustc_session::parse::ParseSess; use rustc_span::source_map::{Span, DUMMY_SP}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; @@ -34,6 +36,7 @@ bitflags::bitflags! { struct Restrictions: u8 { const STMT_EXPR = 1 << 0; const NO_STRUCT_LITERAL = 1 << 1; + const CONST_EXPR = 1 << 2; } } @@ -85,10 +88,14 @@ pub struct Parser<'a> { pub sess: &'a ParseSess, /// The current token. pub token: Token, + /// The spacing for the current token + pub token_spacing: Spacing, /// The previous token. pub prev_token: Token, restrictions: Restrictions, expected_tokens: Vec<TokenType>, + // Important: This must only be advanced from `next_tok` + // to ensure that `token_cursor.num_next_calls` is updated properly token_cursor: TokenCursor, desugar_doc_comments: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is @@ -120,8 +127,10 @@ impl<'a> Drop for Parser<'a> { struct TokenCursor { frame: TokenCursorFrame, stack: Vec<TokenCursorFrame>, - cur_token: Option<TreeAndSpacing>, - collecting: Option<Collecting>, + desugar_doc_comments: bool, + // Counts the number of calls to `next` or `next_desugared`, + // depending on whether `desugar_doc_comments` is set. + num_next_calls: usize, } #[derive(Clone)] @@ -133,40 +142,22 @@ struct TokenCursorFrame { close_delim: bool, } -/// Used to track additional state needed by `collect_tokens` -#[derive(Clone, Debug)] -struct Collecting { - /// Holds the current tokens captured during the most - /// recent call to `collect_tokens` - buf: Vec<TreeAndSpacing>, - /// The depth of the `TokenCursor` stack at the time - /// collection was started. When we encounter a `TokenTree::Delimited`, - /// we want to record the `TokenTree::Delimited` itself, - /// but *not* any of the inner tokens while we are inside - /// the new frame (this would cause us to record duplicate tokens). - /// - /// This `depth` fields tracks stack depth we are recording tokens. - /// Only tokens encountered at this depth will be recorded. See - /// `TokenCursor::next` for more details. - depth: usize, -} - impl TokenCursorFrame { - fn new(span: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { + fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { TokenCursorFrame { delim, span, open_delim: delim == token::NoDelim, - tree_cursor: tts.clone().into_trees(), + tree_cursor: tts.into_trees(), close_delim: delim == token::NoDelim, } } } impl TokenCursor { - fn next(&mut self) -> Token { + fn next(&mut self) -> (Token, Spacing) { loop { - let tree = if !self.frame.open_delim { + let (tree, spacing) = if !self.frame.open_delim { self.frame.open_delim = true; TokenTree::open_tt(self.frame.span, self.frame.delim).into() } else if let Some(tree) = self.frame.tree_cursor.next_with_spacing() { @@ -178,40 +169,24 @@ impl TokenCursor { self.frame = frame; continue; } else { - return Token::new(token::Eof, DUMMY_SP); + (TokenTree::Token(Token::new(token::Eof, DUMMY_SP)), Spacing::Alone) }; - // Don't set an open delimiter as our current token - we want - // to leave it as the full `TokenTree::Delimited` from the previous - // iteration of this loop - if !matches!(tree.0, TokenTree::Token(Token { kind: TokenKind::OpenDelim(_), .. })) { - self.cur_token = Some(tree.clone()); - } - - if let Some(collecting) = &mut self.collecting { - if collecting.depth == self.stack.len() { - debug!( - "TokenCursor::next(): collected {:?} at depth {:?}", - tree, - self.stack.len() - ); - collecting.buf.push(tree.clone()) + match tree { + TokenTree::Token(token) => { + return (token, spacing); } - } - - match tree.0 { - TokenTree::Token(token) => return token, TokenTree::Delimited(sp, delim, tts) => { - let frame = TokenCursorFrame::new(sp, delim, &tts); + let frame = TokenCursorFrame::new(sp, delim, tts); self.stack.push(mem::replace(&mut self.frame, frame)); } } } } - fn next_desugared(&mut self) -> Token { + fn next_desugared(&mut self) -> (Token, Spacing) { let (data, attr_style, sp) = match self.next() { - Token { kind: token::DocComment(_, attr_style, data), span } => { + (Token { kind: token::DocComment(_, attr_style, data), span }, _) => { (data, attr_style, span) } tok => return tok, @@ -249,7 +224,7 @@ impl TokenCursor { TokenCursorFrame::new( delim_span, token::NoDelim, - &if attr_style == AttrStyle::Inner { + if attr_style == AttrStyle::Inner { [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] .iter() .cloned() @@ -351,14 +326,15 @@ impl<'a> Parser<'a> { let mut parser = Parser { sess, token: Token::dummy(), + token_spacing: Spacing::Alone, prev_token: Token::dummy(), restrictions: Restrictions::empty(), expected_tokens: Vec::new(), token_cursor: TokenCursor { - frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, &tokens), + frame: TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens), stack: Vec::new(), - cur_token: None, - collecting: None, + num_next_calls: 0, + desugar_doc_comments, }, desugar_doc_comments, unmatched_angle_bracket_count: 0, @@ -375,17 +351,18 @@ impl<'a> Parser<'a> { parser } - fn next_tok(&mut self, fallback_span: Span) -> Token { - let mut next = if self.desugar_doc_comments { + fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) { + let (mut next, spacing) = if self.desugar_doc_comments { self.token_cursor.next_desugared() } else { self.token_cursor.next() }; + self.token_cursor.num_next_calls += 1; if next.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. next.span = fallback_span.with_ctxt(next.span.ctxt()); } - next + (next, spacing) } pub fn unexpected<T>(&mut self) -> PResult<'a, T> { @@ -546,9 +523,13 @@ impl<'a> Parser<'a> { self.check_or_expected(self.token.can_begin_const_arg(), TokenType::Const) } - fn check_inline_const(&mut self) -> bool { - self.check_keyword(kw::Const) - && self.look_ahead(1, |t| t == &token::OpenDelim(DelimToken::Brace)) + fn check_inline_const(&self, dist: usize) -> bool { + self.is_keyword_ahead(dist, &[kw::Const]) + && self.look_ahead(dist + 1, |t| match t.kind { + token::Interpolated(ref nt) => matches!(**nt, token::NtBlock(..)), + token::OpenDelim(DelimToken::Brace) => true, + _ => false, + }) } /// Checks to see if the next token is either `+` or `+=`. @@ -573,7 +554,9 @@ impl<'a> Parser<'a> { let first_span = self.sess.source_map().start_point(self.token.span); let second_span = self.token.span.with_lo(first_span.hi()); self.token = Token::new(first, first_span); - self.bump_with(Token::new(second, second_span)); + // Use the spacing of the glued token as the spacing + // of the unglued second token. + self.bump_with((Token::new(second, second_span), self.token_spacing)); true } _ => { @@ -805,7 +788,7 @@ impl<'a> Parser<'a> { } /// Advance the parser by one token using provided token as the next one. - fn bump_with(&mut self, next_token: Token) { + fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { // Bumping after EOF is a bad sign, usually an infinite loop. if self.prev_token.kind == TokenKind::Eof { let msg = "attempted to bump the parser past EOF (may be stuck in a loop)"; @@ -814,6 +797,7 @@ impl<'a> Parser<'a> { // Update the current and previous tokens. self.prev_token = mem::replace(&mut self.token, next_token); + self.token_spacing = next_spacing; // Diagnostics. self.expected_tokens.clear(); @@ -833,15 +817,15 @@ impl<'a> Parser<'a> { } let frame = &self.token_cursor.frame; - looker(&match frame.tree_cursor.look_ahead(dist - 1) { + match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { - TokenTree::Token(token) => token, + TokenTree::Token(token) => looker(token), TokenTree::Delimited(dspan, delim, _) => { - Token::new(token::OpenDelim(delim), dspan.open) + looker(&Token::new(token::OpenDelim(*delim), dspan.open)) } }, - None => Token::new(token::CloseDelim(frame.delim), frame.span.close), - }) + None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)), + } } /// Returns whether any of the given keywords are `dist` tokens ahead of the current one. @@ -881,7 +865,7 @@ impl<'a> Parser<'a> { } /// Parses inline const expressions. - fn parse_const_expr(&mut self, span: Span) -> PResult<'a, P<Expr>> { + fn parse_const_block(&mut self, span: Span) -> PResult<'a, P<Expr>> { self.sess.gated_spans.gate(sym::inline_const, span); self.eat_keyword(kw::Const); let blk = self.parse_block()?; @@ -984,13 +968,27 @@ impl<'a> Parser<'a> { pub(crate) fn parse_token_tree(&mut self) -> TokenTree { match self.token.kind { token::OpenDelim(..) => { - let frame = mem::replace( - &mut self.token_cursor.frame, - self.token_cursor.stack.pop().unwrap(), - ); - self.token = Token::new(TokenKind::CloseDelim(frame.delim), frame.span.close); + let depth = self.token_cursor.stack.len(); + + // We keep advancing the token cursor until we hit + // the matching `CloseDelim` token. + while !(depth == self.token_cursor.stack.len() + && matches!(self.token.kind, token::CloseDelim(_))) + { + // Advance one token at a time, so `TokenCursor::next()` + // can capture these tokens if necessary. + self.bump(); + } + // We are still inside the frame corresponding + // to the delimited stream we captured, so grab + // the tokens from this frame. + let frame = &self.token_cursor.frame; + let stream = frame.tree_cursor.stream.clone(); + let span = frame.span; + let delim = frame.delim; + // Consume close delimiter self.bump(); - TokenTree::Delimited(frame.span, frame.delim, frame.tree_cursor.stream) + TokenTree::Delimited(span, delim, stream) } token::CloseDelim(_) | token::Eof => unreachable!(), _ => { @@ -1181,8 +1179,9 @@ impl<'a> Parser<'a> { /// Records all tokens consumed by the provided callback, /// including the current token. These tokens are collected - /// into a `TokenStream`, and returned along with the result - /// of the callback. + /// into a `LazyTokenStream`, and returned along with the result + /// of the callback. The returned `LazyTokenStream` will be `None` + /// if not tokens were captured. /// /// Note: If your callback consumes an opening delimiter /// (including the case where you call `collect_tokens` @@ -1198,79 +1197,61 @@ impl<'a> Parser<'a> { pub fn collect_tokens<R>( &mut self, f: impl FnOnce(&mut Self) -> PResult<'a, R>, - ) -> PResult<'a, (R, TokenStream)> { - // Record all tokens we parse when parsing this item. - let tokens: Vec<TreeAndSpacing> = self.token_cursor.cur_token.clone().into_iter().collect(); - debug!("collect_tokens: starting with {:?}", tokens); - - // We need special handling for the case where `collect_tokens` is called - // on an opening delimeter (e.g. '('). At this point, we have already pushed - // a new frame - however, we want to record the original `TokenTree::Delimited`, - // for consistency with the case where we start recording one token earlier. - // See `TokenCursor::next` to see how `cur_token` is set up. - let prev_depth = - if matches!(self.token_cursor.cur_token, Some((TokenTree::Delimited(..), _))) { - if self.token_cursor.stack.is_empty() { - // There is nothing below us in the stack that - // the function could consume, so the only thing it can legally - // capture is the entire contents of the current frame. - return Ok((f(self)?, TokenStream::new(tokens))); - } - // We have already recorded the full `TokenTree::Delimited` when we created - // our `tokens` vector at the start of this function. We are now inside - // a new frame corresponding to the `TokenTree::Delimited` we already recoreded. - // We don't want to record any of the tokens inside this frame, since they - // will be duplicates of the tokens nested inside the `TokenTree::Delimited`. - // Therefore, we set our recording depth to the *previous* frame. This allows - // us to record a sequence like: `(foo).bar()`: the `(foo)` will be recored - // as our initial `cur_token`, while the `.bar()` will be recored after we - // pop the `(foo)` frame. - self.token_cursor.stack.len() - 1 - } else { - self.token_cursor.stack.len() - }; - let prev_collecting = - self.token_cursor.collecting.replace(Collecting { buf: tokens, depth: prev_depth }); + ) -> PResult<'a, (R, Option<LazyTokenStream>)> { + let start_token = (self.token.clone(), self.token_spacing); + let cursor_snapshot = self.token_cursor.clone(); - let ret = f(self); + let ret = f(self)?; - let mut collected_tokens = if let Some(collecting) = self.token_cursor.collecting.take() { - collecting.buf - } else { - let msg = "our vector went away?"; - debug!("collect_tokens: {}", msg); - self.sess.span_diagnostic.delay_span_bug(self.token.span, &msg); - // This can happen due to a bad interaction of two unrelated recovery mechanisms - // with mismatched delimiters *and* recovery lookahead on the likely typo - // `pub ident(` (#62895, different but similar to the case above). - return Ok((ret?, TokenStream::default())); - }; + // We didn't capture any tokens + let num_calls = self.token_cursor.num_next_calls - cursor_snapshot.num_next_calls; + if num_calls == 0 { + return Ok((ret, None)); + } + + // Produces a `TokenStream` on-demand. Using `cursor_snapshot` + // and `num_calls`, we can reconstruct the `TokenStream` seen + // by the callback. This allows us to avoid producing a `TokenStream` + // if it is never needed - for example, a captured `macro_rules!` + // argument that is never passed to a proc macro. + // + // This also makes `Parser` very cheap to clone, since + // there is no intermediate collection buffer to clone. + struct LazyTokenStreamImpl { + start_token: (Token, Spacing), + cursor_snapshot: TokenCursor, + num_calls: usize, + desugar_doc_comments: bool, + } + impl CreateTokenStream for LazyTokenStreamImpl { + fn create_token_stream(&self) -> TokenStream { + // The token produced by the final call to `next` or `next_desugared` + // was not actually consumed by the callback. The combination + // of chaining the initial token and using `take` produces the desired + // result - we produce an empty `TokenStream` if no calls were made, + // and omit the final token otherwise. + let mut cursor_snapshot = self.cursor_snapshot.clone(); + let tokens = std::iter::once(self.start_token.clone()) + .chain((0..self.num_calls).map(|_| { + if self.desugar_doc_comments { + cursor_snapshot.next_desugared() + } else { + cursor_snapshot.next() + } + })) + .take(self.num_calls); - debug!("collect_tokens: got raw tokens {:?}", collected_tokens); - - // If we're not at EOF our current token wasn't actually consumed by - // `f`, but it'll still be in our list that we pulled out. In that case - // put it back. - let extra_token = if self.token != token::Eof { collected_tokens.pop() } else { None }; - - if let Some(mut collecting) = prev_collecting { - // If we were previously collecting at the same depth, - // then the previous call to `collect_tokens` needs to see - // the tokens we just recorded. - // - // If we were previously recording at an lower `depth`, - // then the previous `collect_tokens` call already recorded - // this entire frame in the form of a `TokenTree::Delimited`, - // so there is nothing else for us to do. - if collecting.depth == prev_depth { - collecting.buf.extend(collected_tokens.iter().cloned()); - collecting.buf.extend(extra_token); - debug!("collect_tokens: updating previous buf to {:?}", collecting); + make_token_stream(tokens) } - self.token_cursor.collecting = Some(collecting) } - Ok((ret?, TokenStream::new(collected_tokens))) + let lazy_impl = LazyTokenStreamImpl { + start_token, + cursor_snapshot, + num_calls, + desugar_doc_comments: self.desugar_doc_comments, + }; + Ok((ret, Some(LazyTokenStream::new(lazy_impl)))) } /// `::{` or `::*` @@ -1319,3 +1300,41 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, sess: &Pa } } } + +/// Converts a flattened iterator of tokens (including open and close delimiter tokens) +/// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair +/// of open and close delims. +fn make_token_stream(tokens: impl Iterator<Item = (Token, Spacing)>) -> TokenStream { + #[derive(Debug)] + struct FrameData { + open: Span, + inner: Vec<(TokenTree, Spacing)>, + } + let mut stack = vec![FrameData { open: DUMMY_SP, inner: vec![] }]; + for (token, spacing) in tokens { + match token { + Token { kind: TokenKind::OpenDelim(_), span } => { + stack.push(FrameData { open: span, inner: vec![] }); + } + Token { kind: TokenKind::CloseDelim(delim), span } => { + let frame_data = stack.pop().expect("Token stack was empty!"); + let dspan = DelimSpan::from_pair(frame_data.open, span); + let stream = TokenStream::new(frame_data.inner); + let delimited = TokenTree::Delimited(dspan, delim, stream); + stack + .last_mut() + .unwrap_or_else(|| panic!("Bottom token frame is missing for tokens!")) + .inner + .push((delimited, Spacing::Alone)); + } + token => stack + .last_mut() + .expect("Bottom token frame is missing!") + .inner + .push((TokenTree::Token(token), spacing)), + } + } + let final_buf = stack.pop().expect("Missing final buf!"); + assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack); + TokenStream::new(final_buf.inner) +} diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 15660fd574c..ab88362dad9 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -38,16 +38,13 @@ impl<'a> Parser<'a> { }, NonterminalKind::Block => match token.kind { token::OpenDelim(token::Brace) => true, - token::Interpolated(ref nt) => match **nt { - token::NtItem(_) + token::Interpolated(ref nt) => !matches!(**nt, token::NtItem(_) | token::NtPat(_) | token::NtTy(_) | token::NtIdent(..) | token::NtMeta(_) | token::NtPath(_) - | token::NtVis(_) => false, // none of these may start with '{'. - _ => true, - }, + | token::NtVis(_)), _ => false, }, NonterminalKind::Path | NonterminalKind::Meta => match token.kind { @@ -76,17 +73,14 @@ impl<'a> Parser<'a> { }, NonterminalKind::Lifetime => match token.kind { token::Lifetime(_) => true, - token::Interpolated(ref nt) => match **nt { - token::NtLifetime(_) | token::NtTT(_) => true, - _ => false, - }, + token::Interpolated(ref nt) => { + matches!(**nt, token::NtLifetime(_) | token::NtTT(_)) + } _ => false, }, - NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => match token.kind - { - token::CloseDelim(_) => false, - _ => true, - }, + NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => { + !matches!(token.kind, token::CloseDelim(_)) + } } } @@ -103,7 +97,7 @@ impl<'a> Parser<'a> { // If we captured tokens during parsing (due to outer attributes), // use those. if item.tokens.is_none() { - item.tokens = Some(tokens); + item.tokens = tokens; } token::NtItem(item) } @@ -115,7 +109,7 @@ impl<'a> Parser<'a> { let (mut block, tokens) = self.collect_tokens(|this| this.parse_block())?; // We have have eaten an NtBlock, which could already have tokens if block.tokens.is_none() { - block.tokens = Some(tokens); + block.tokens = tokens; } token::NtBlock(block) } @@ -124,7 +118,7 @@ impl<'a> Parser<'a> { match stmt { Some(mut s) => { if s.tokens.is_none() { - s.tokens = Some(tokens); + s.tokens = tokens; } token::NtStmt(s) } @@ -137,7 +131,7 @@ impl<'a> Parser<'a> { let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?; // We have have eaten an NtPat, which could already have tokens if pat.tokens.is_none() { - pat.tokens = Some(tokens); + pat.tokens = tokens; } token::NtPat(pat) } @@ -146,7 +140,7 @@ impl<'a> Parser<'a> { // If we captured tokens during parsing (due to outer attributes), // use those. if expr.tokens.is_none() { - expr.tokens = Some(tokens); + expr.tokens = tokens; } token::NtExpr(expr) } @@ -155,7 +149,7 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_literal_maybe_minus())?; // We have have eaten a nonterminal, which could already have tokens if lit.tokens.is_none() { - lit.tokens = Some(tokens); + lit.tokens = tokens; } token::NtLiteral(lit) } @@ -163,7 +157,7 @@ impl<'a> Parser<'a> { let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?; // We have an eaten an NtTy, which could already have tokens if ty.tokens.is_none() { - ty.tokens = Some(tokens); + ty.tokens = tokens; } token::NtTy(ty) } @@ -183,15 +177,15 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_path(PathStyle::Type))?; // We have have eaten an NtPath, which could already have tokens if path.tokens.is_none() { - path.tokens = Some(tokens); + path.tokens = tokens; } token::NtPath(path) } NonterminalKind::Meta => { - let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?; + let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item(false))?; // We may have eaten a nonterminal, which could already have tokens if attr.tokens.is_none() { - attr.tokens = Some(tokens); + attr.tokens = tokens; } token::NtMeta(P(attr)) } @@ -201,7 +195,7 @@ impl<'a> Parser<'a> { self.collect_tokens(|this| this.parse_visibility(FollowedByType::Yes))?; // We may have etan an `NtVis`, which could already have tokens if vis.tokens.is_none() { - vis.tokens = Some(tokens); + vis.tokens = tokens; } token::NtVis(vis) } diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 15db2066a30..196790a0ab3 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -149,8 +149,10 @@ impl<'a> Parser<'a> { /// Note that there are more tokens such as `@` for which we know that the `|` /// is an illegal parse. However, the user's intent is less clear in that case. fn recover_trailing_vert(&mut self, lo: Option<Span>) -> bool { - let is_end_ahead = self.look_ahead(1, |token| match &token.uninterpolate().kind { - token::FatArrow // e.g. `a | => 0,`. + let is_end_ahead = self.look_ahead(1, |token| { + matches!( + &token.uninterpolate().kind, + token::FatArrow // e.g. `a | => 0,`. | token::Ident(kw::If, false) // e.g. `a | if expr`. | token::Eq // e.g. `let a | = 0`. | token::Semi // e.g. `let a |;`. @@ -158,8 +160,8 @@ impl<'a> Parser<'a> { | token::Comma // e.g. `let (a |,)`. | token::CloseDelim(token::Bracket) // e.g. `let [a | ]`. | token::CloseDelim(token::Paren) // e.g. `let (a | )`. - | token::CloseDelim(token::Brace) => true, // e.g. `let A { f: a | }`. - _ => false, + | token::CloseDelim(token::Brace) // e.g. `let A { f: a | }`. + ) }); match (is_end_ahead, &self.token.kind) { (true, token::BinOp(token::Or) | token::OrOr) => { @@ -313,9 +315,15 @@ impl<'a> Parser<'a> { let pat = self.parse_pat_with_range_pat(false, None)?; self.sess.gated_spans.gate(sym::box_patterns, lo.to(self.prev_token.span)); PatKind::Box(pat) - } else if self.check_inline_const() { + } else if self.check_inline_const(0) { // Parse `const pat` - PatKind::Lit(self.parse_const_expr(lo.to(self.token.span))?) + let const_expr = self.parse_const_block(lo.to(self.token.span))?; + + if let Some(re) = self.parse_range_end() { + self.parse_pat_range_begin_with(const_expr, re)? + } else { + PatKind::Lit(const_expr) + } } else if self.can_be_ident_pat() { // Parse `ident @ pat` // This can give false positives and parse nullary enums, @@ -717,16 +725,19 @@ impl<'a> Parser<'a> { /// Is the token `dist` away from the current suitable as the start of a range patterns end? fn is_pat_range_end_start(&self, dist: usize) -> bool { - self.look_ahead(dist, |t| { - t.is_path_start() // e.g. `MY_CONST`; + self.check_inline_const(dist) + || self.look_ahead(dist, |t| { + t.is_path_start() // e.g. `MY_CONST`; || t.kind == token::Dot // e.g. `.5` for recovery; || t.can_begin_literal_maybe_minus() // e.g. `42`. || t.is_whole_expr() - }) + }) } fn parse_pat_range_end(&mut self) -> PResult<'a, P<Expr>> { - if self.check_path() { + if self.check_inline_const(0) { + self.parse_const_block(self.token.span) + } else if self.check_path() { let lo = self.token.span; let (qself, path) = if self.eat_lt() { // Parse a qualified path @@ -757,14 +768,12 @@ impl<'a> Parser<'a> { && !self.token.is_path_segment_keyword() // Avoid e.g. `Self` as it is a path. // Avoid `in`. Due to recovery in the list parser this messes with `for ( $pat in $expr )`. && !self.token.is_keyword(kw::In) - && self.look_ahead(1, |t| match t.kind { // Try to do something more complex? - token::OpenDelim(token::Paren) // A tuple struct pattern. + // Try to do something more complex? + && self.look_ahead(1, |t| !matches!(t.kind, token::OpenDelim(token::Paren) // A tuple struct pattern. | token::OpenDelim(token::Brace) // A struct pattern. | token::DotDotDot | token::DotDotEq | token::DotDot // A range pattern. | token::ModSep // A tuple / struct variant pattern. - | token::Not => false, // A macro expanding to a pattern. - _ => true, - }) + | token::Not)) // A macro expanding to a pattern. } /// Parses `ident` or `ident @ pat`. diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 66ce015d02e..79e73749038 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -187,12 +187,14 @@ impl<'a> Parser<'a> { pub(super) fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> { let ident = self.parse_path_segment_ident()?; - let is_args_start = |token: &Token| match token.kind { - token::Lt - | token::BinOp(token::Shl) - | token::OpenDelim(token::Paren) - | token::LArrow => true, - _ => false, + let is_args_start = |token: &Token| { + matches!( + token.kind, + token::Lt + | token::BinOp(token::Shl) + | token::OpenDelim(token::Paren) + | token::LArrow + ) }; let check_args_start = |this: &mut Self| { this.expected_tokens.extend_from_slice(&[ @@ -397,6 +399,13 @@ impl<'a> Parser<'a> { while let Some(arg) = self.parse_angle_arg()? { args.push(arg); if !self.eat(&token::Comma) { + if !self.token.kind.should_end_const_arg() { + if self.handle_ambiguous_unbraced_const_arg(&mut args)? { + // We've managed to (partially) recover, so continue trying to parse + // arguments. + continue; + } + } break; } } @@ -476,41 +485,50 @@ impl<'a> Parser<'a> { Ok(self.mk_ty(span, ast::TyKind::Err)) } + /// We do not permit arbitrary expressions as const arguments. They must be one of: + /// - An expression surrounded in `{}`. + /// - A literal. + /// - A numeric literal prefixed by `-`. + pub(super) fn expr_is_valid_const_arg(&self, expr: &P<rustc_ast::Expr>) -> bool { + match &expr.kind { + ast::ExprKind::Block(_, _) | ast::ExprKind::Lit(_) => true, + ast::ExprKind::Unary(ast::UnOp::Neg, expr) => match &expr.kind { + ast::ExprKind::Lit(_) => true, + _ => false, + }, + _ => false, + } + } + /// Parse a generic argument in a path segment. /// This does not include constraints, e.g., `Item = u8`, which is handled in `parse_angle_arg`. fn parse_generic_arg(&mut self) -> PResult<'a, Option<GenericArg>> { + let start = self.token.span; let arg = if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) { // Parse lifetime argument. GenericArg::Lifetime(self.expect_lifetime()) } else if self.check_const_arg() { // Parse const argument. - let expr = if let token::OpenDelim(token::Brace) = self.token.kind { + let value = if let token::OpenDelim(token::Brace) = self.token.kind { self.parse_block_expr( None, self.token.span, BlockCheckMode::Default, ast::AttrVec::new(), )? - } else if self.token.is_ident() { - // FIXME(const_generics): to distinguish between idents for types and consts, - // we should introduce a GenericArg::Ident in the AST and distinguish when - // lowering to the HIR. For now, idents for const args are not permitted. - if self.token.is_bool_lit() { - self.parse_literal_maybe_minus()? - } else { - let span = self.token.span; - let msg = "identifiers may currently not be used for const generics"; - self.struct_span_err(span, msg).emit(); - let block = self.mk_block_err(span); - self.mk_expr(span, ast::ExprKind::Block(block, None), ast::AttrVec::new()) - } } else { - self.parse_literal_maybe_minus()? + self.handle_unambiguous_unbraced_const_arg()? }; - GenericArg::Const(AnonConst { id: ast::DUMMY_NODE_ID, value: expr }) + GenericArg::Const(AnonConst { id: ast::DUMMY_NODE_ID, value }) } else if self.check_type() { // Parse type argument. - GenericArg::Type(self.parse_ty()?) + match self.parse_ty() { + Ok(ty) => GenericArg::Type(ty), + Err(err) => { + // Try to recover from possible `const` arg without braces. + return self.recover_const_arg(start, err).map(Some); + } + } } else { return Ok(None); }; diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs index d42a786a18f..7a6ebca4e15 100644 --- a/compiler/rustc_parse/src/parser/ty.rs +++ b/compiler/rustc_parse/src/parser/ty.rs @@ -265,7 +265,19 @@ impl<'a> Parser<'a> { /// Parses an array (`[TYPE; EXPR]`) or slice (`[TYPE]`) type. /// The opening `[` bracket is already eaten. fn parse_array_or_slice_ty(&mut self) -> PResult<'a, TyKind> { - let elt_ty = self.parse_ty()?; + let elt_ty = match self.parse_ty() { + Ok(ty) => ty, + Err(mut err) + if self.look_ahead(1, |t| t.kind == token::CloseDelim(token::Bracket)) + | self.look_ahead(1, |t| t.kind == token::Semi) => + { + // Recover from `[LIT; EXPR]` and `[LIT]` + self.bump(); + err.emit(); + self.mk_ty(self.prev_token.span, TyKind::Err) + } + Err(err) => return Err(err), + }; let ty = if self.eat(&token::Semi) { TyKind::Array(elt_ty, self.parse_anon_const_expr()?) } else { diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 1acaa4c6eff..7679582f881 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -85,6 +85,10 @@ impl CheckAttrVisitor<'tcx> { self.check_export_name(&attr, span, target) } else if self.tcx.sess.check_name(attr, sym::rustc_args_required_const) { self.check_rustc_args_required_const(&attr, span, target, item) + } else if self.tcx.sess.check_name(attr, sym::allow_internal_unstable) { + self.check_allow_internal_unstable(&attr, span, target, &attrs) + } else if self.tcx.sess.check_name(attr, sym::rustc_allow_const_fn_unstable) { + self.check_rustc_allow_const_fn_unstable(hir_id, &attr, span, target) } else { // lint-only checks if self.tcx.sess.check_name(attr, sym::cold) { @@ -104,7 +108,7 @@ impl CheckAttrVisitor<'tcx> { return; } - if matches!(target, Target::Fn | Target::Method(_) | Target::ForeignFn) { + if matches!(target, Target::Closure | Target::Fn | Target::Method(_) | Target::ForeignFn) { self.tcx.ensure().codegen_fn_attrs(self.tcx.hir().local_def_id(hir_id)); } @@ -195,7 +199,7 @@ impl CheckAttrVisitor<'tcx> { /// Checks if the `#[non_exhaustive]` attribute on an `item` is valid. Returns `true` if valid. fn check_non_exhaustive(&self, attr: &Attribute, span: &Span, target: Target) -> bool { match target { - Target::Struct | Target::Enum => true, + Target::Struct | Target::Enum | Target::Variant => true, _ => { struct_span_err!( self.tcx.sess, @@ -587,6 +591,9 @@ impl CheckAttrVisitor<'tcx> { for hint in &hints { let (article, allowed_targets) = match hint.name_or_empty() { + _ if !matches!(target, Target::Struct | Target::Enum | Target::Union) => { + ("a", "struct, enum, or union") + } name @ sym::C | name @ sym::align => { is_c |= name == sym::C; match target { @@ -652,12 +659,16 @@ impl CheckAttrVisitor<'tcx> { } _ => continue, }; - self.emit_repr_error( + + struct_span_err!( + self.tcx.sess, hint.span(), - *span, - &format!("attribute should be applied to {}", allowed_targets), - &format!("not {} {}", article, allowed_targets), + E0517, + "{}", + &format!("attribute should be applied to {} {}", article, allowed_targets) ) + .span_label(*span, &format!("not {} {}", article, allowed_targets)) + .emit(); } // Just point at all repr hints if there are any incompatibilities. @@ -703,64 +714,63 @@ impl CheckAttrVisitor<'tcx> { } } - fn emit_repr_error( - &self, - hint_span: Span, - label_span: Span, - hint_message: &str, - label_message: &str, - ) { - struct_span_err!(self.tcx.sess, hint_span, E0517, "{}", hint_message) - .span_label(label_span, label_message) - .emit(); - } - - fn check_stmt_attributes(&self, stmt: &hir::Stmt<'_>) { - // When checking statements ignore expressions, they will be checked later - if let hir::StmtKind::Local(ref l) = stmt.kind { - self.check_attributes(l.hir_id, &l.attrs, &stmt.span, Target::Statement, None); - for attr in l.attrs.iter() { - if self.tcx.sess.check_name(attr, sym::repr) { - self.emit_repr_error( - attr.span, - stmt.span, - "attribute should not be applied to a statement", - "not a struct, enum, or union", - ); - } + fn check_used(&self, attrs: &'hir [Attribute], target: Target) { + for attr in attrs { + if self.tcx.sess.check_name(attr, sym::used) && target != Target::Static { + self.tcx + .sess + .span_err(attr.span, "attribute must be applied to a `static` variable"); } } } - fn check_expr_attributes(&self, expr: &hir::Expr<'_>) { - let target = match expr.kind { - hir::ExprKind::Closure(..) => Target::Closure, - _ => Target::Expression, - }; - self.check_attributes(expr.hir_id, &expr.attrs, &expr.span, target, None); - for attr in expr.attrs.iter() { - if self.tcx.sess.check_name(attr, sym::repr) { - self.emit_repr_error( - attr.span, - expr.span, - "attribute should not be applied to an expression", - "not defining a struct, enum, or union", - ); + /// Outputs an error for `#[allow_internal_unstable]` which can only be applied to macros. + /// (Allows proc_macro functions) + fn check_allow_internal_unstable( + &self, + attr: &Attribute, + span: &Span, + target: Target, + attrs: &[Attribute], + ) -> bool { + debug!("Checking target: {:?}", target); + if target == Target::Fn { + for attr in attrs { + if self.tcx.sess.is_proc_macro_attr(attr) { + debug!("Is proc macro attr"); + return true; + } } + debug!("Is not proc macro attr"); } - if target == Target::Closure { - self.tcx.ensure().codegen_fn_attrs(self.tcx.hir().local_def_id(expr.hir_id)); - } + self.tcx + .sess + .struct_span_err(attr.span, "attribute should be applied to a macro") + .span_label(*span, "not a macro") + .emit(); + false } - fn check_used(&self, attrs: &'hir [Attribute], target: Target) { - for attr in attrs { - if self.tcx.sess.check_name(attr, sym::used) && target != Target::Static { - self.tcx - .sess - .span_err(attr.span, "attribute must be applied to a `static` variable"); + /// Outputs an error for `#[allow_internal_unstable]` which can only be applied to macros. + /// (Allows proc_macro functions) + fn check_rustc_allow_const_fn_unstable( + &self, + hir_id: HirId, + attr: &Attribute, + span: &Span, + target: Target, + ) -> bool { + if let Target::Fn | Target::Method(_) = target { + if self.tcx.is_const_fn_raw(self.tcx.hir().local_def_id(hir_id)) { + return true; } } + self.tcx + .sess + .struct_span_err(attr.span, "attribute should be applied to `const fn`") + .span_label(*span, "not a `const fn`") + .emit(); + false } } @@ -808,14 +818,32 @@ impl Visitor<'tcx> for CheckAttrVisitor<'tcx> { } fn visit_stmt(&mut self, stmt: &'tcx hir::Stmt<'tcx>) { - self.check_stmt_attributes(stmt); + // When checking statements ignore expressions, they will be checked later. + if let hir::StmtKind::Local(ref l) = stmt.kind { + self.check_attributes(l.hir_id, &l.attrs, &stmt.span, Target::Statement, None); + } intravisit::walk_stmt(self, stmt) } fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) { - self.check_expr_attributes(expr); + let target = match expr.kind { + hir::ExprKind::Closure(..) => Target::Closure, + _ => Target::Expression, + }; + + self.check_attributes(expr.hir_id, &expr.attrs, &expr.span, target, None); intravisit::walk_expr(self, expr) } + + fn visit_variant( + &mut self, + variant: &'tcx hir::Variant<'tcx>, + generics: &'tcx hir::Generics<'tcx>, + item_id: HirId, + ) { + self.check_attributes(variant.id, variant.attrs, &variant.span, Target::Variant, None); + intravisit::walk_variant(self, variant, generics, item_id) + } } fn is_c_like_enum(item: &Item<'_>) -> bool { diff --git a/compiler/rustc_passes/src/check_const.rs b/compiler/rustc_passes/src/check_const.rs index dd0bcbf208d..b24c62b971a 100644 --- a/compiler/rustc_passes/src/check_const.rs +++ b/compiler/rustc_passes/src/check_const.rs @@ -87,7 +87,7 @@ impl<'tcx> CheckConstVisitor<'tcx> { let is_feature_allowed = |feature_gate| { // All features require that the corresponding gate be enabled, - // even if the function has `#[allow_internal_unstable(the_gate)]`. + // even if the function has `#[rustc_allow_const_fn_unstable(the_gate)]`. if !tcx.features().enabled(feature_gate) { return false; } @@ -105,8 +105,8 @@ impl<'tcx> CheckConstVisitor<'tcx> { } // However, we cannot allow stable `const fn`s to use unstable features without an explicit - // opt-in via `allow_internal_unstable`. - attr::allow_internal_unstable(&tcx.sess, &tcx.get_attrs(def_id)) + // opt-in via `rustc_allow_const_fn_unstable`. + attr::rustc_allow_const_fn_unstable(&tcx.sess, &tcx.get_attrs(def_id)) .map_or(false, |mut features| features.any(|name| name == feature_gate)) }; diff --git a/compiler/rustc_passes/src/hir_id_validator.rs b/compiler/rustc_passes/src/hir_id_validator.rs index 24695f5cdfa..6d1a5fcc10b 100644 --- a/compiler/rustc_passes/src/hir_id_validator.rs +++ b/compiler/rustc_passes/src/hir_id_validator.rs @@ -163,4 +163,17 @@ impl<'a, 'hir> intravisit::Visitor<'hir> for HirIdValidator<'a, 'hir> { // we are currently in. So for those it's correct that they have a // different owner. } + + fn visit_generic_param(&mut self, param: &'hir hir::GenericParam<'hir>) { + if let hir::GenericParamKind::Type { + synthetic: Some(hir::SyntheticTyParamKind::ImplTrait), + .. + } = param.kind + { + // Synthetic impl trait parameters are owned by the node of the desugared type. + // This means it is correct for them to have a different owner. + } else { + intravisit::walk_generic_param(self, param); + } + } } diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs index ae810b9e79a..7288015e170 100644 --- a/compiler/rustc_passes/src/liveness.rs +++ b/compiler/rustc_passes/src/liveness.rs @@ -1174,7 +1174,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> { } } hir::InlineAsmOperand::InOut { expr, .. } => { - succ = self.write_place(expr, succ, ACC_READ | ACC_WRITE); + succ = self.write_place(expr, succ, ACC_READ | ACC_WRITE | ACC_USE); } hir::InlineAsmOperand::SplitInOut { out_expr, .. } => { if let Some(expr) = out_expr { diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs index 8f93bce6e99..4c0941120a6 100644 --- a/compiler/rustc_privacy/src/lib.rs +++ b/compiler/rustc_privacy/src/lib.rs @@ -1,6 +1,9 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![feature(in_band_lifetimes)] #![feature(nll)] +#![feature(or_patterns)] +#![feature(control_flow_enum)] +#![feature(try_blocks)] #![recursion_limit = "256"] use rustc_attr as attr; @@ -14,16 +17,18 @@ use rustc_hir::{AssocItemKind, HirIdSet, Node, PatKind}; use rustc_middle::bug; use rustc_middle::hir::map::Map; use rustc_middle::middle::privacy::{AccessLevel, AccessLevels}; +use rustc_middle::span_bug; use rustc_middle::ty::fold::TypeVisitor; use rustc_middle::ty::query::Providers; use rustc_middle::ty::subst::InternalSubsts; use rustc_middle::ty::{self, GenericParamDefKind, TraitRef, Ty, TyCtxt, TypeFoldable}; use rustc_session::lint; use rustc_span::hygiene::Transparency; -use rustc_span::symbol::{kw, sym, Ident}; +use rustc_span::symbol::{kw, Ident}; use rustc_span::Span; use std::marker::PhantomData; +use std::ops::ControlFlow; use std::{cmp, fmt, mem}; //////////////////////////////////////////////////////////////////////////////// @@ -46,7 +51,12 @@ trait DefIdVisitor<'tcx> { fn skip_assoc_tys(&self) -> bool { false } - fn visit_def_id(&mut self, def_id: DefId, kind: &str, descr: &dyn fmt::Display) -> bool; + fn visit_def_id( + &mut self, + def_id: DefId, + kind: &str, + descr: &dyn fmt::Display, + ) -> ControlFlow<()>; /// Not overridden, but used to actually visit types and traits. fn skeleton(&mut self) -> DefIdVisitorSkeleton<'_, 'tcx, Self> { @@ -56,13 +66,13 @@ trait DefIdVisitor<'tcx> { dummy: Default::default(), } } - fn visit(&mut self, ty_fragment: impl TypeFoldable<'tcx>) -> bool { + fn visit(&mut self, ty_fragment: impl TypeFoldable<'tcx>) -> ControlFlow<()> { ty_fragment.visit_with(&mut self.skeleton()) } - fn visit_trait(&mut self, trait_ref: TraitRef<'tcx>) -> bool { + fn visit_trait(&mut self, trait_ref: TraitRef<'tcx>) -> ControlFlow<()> { self.skeleton().visit_trait(trait_ref) } - fn visit_predicates(&mut self, predicates: ty::GenericPredicates<'tcx>) -> bool { + fn visit_predicates(&mut self, predicates: ty::GenericPredicates<'tcx>) -> ControlFlow<()> { self.skeleton().visit_predicates(predicates) } } @@ -77,25 +87,25 @@ impl<'tcx, V> DefIdVisitorSkeleton<'_, 'tcx, V> where V: DefIdVisitor<'tcx> + ?Sized, { - fn visit_trait(&mut self, trait_ref: TraitRef<'tcx>) -> bool { + fn visit_trait(&mut self, trait_ref: TraitRef<'tcx>) -> ControlFlow<()> { let TraitRef { def_id, substs } = trait_ref; - self.def_id_visitor.visit_def_id(def_id, "trait", &trait_ref.print_only_trait_path()) - || (!self.def_id_visitor.shallow() && substs.visit_with(self)) + self.def_id_visitor.visit_def_id(def_id, "trait", &trait_ref.print_only_trait_path())?; + if self.def_id_visitor.shallow() { ControlFlow::CONTINUE } else { substs.visit_with(self) } } - fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> bool { + fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ControlFlow<()> { match predicate.skip_binders() { ty::PredicateAtom::Trait(ty::TraitPredicate { trait_ref }, _) => { self.visit_trait(trait_ref) } ty::PredicateAtom::Projection(ty::ProjectionPredicate { projection_ty, ty }) => { - ty.visit_with(self) - || self.visit_trait(projection_ty.trait_ref(self.def_id_visitor.tcx())) + ty.visit_with(self)?; + self.visit_trait(projection_ty.trait_ref(self.def_id_visitor.tcx())) } ty::PredicateAtom::TypeOutlives(ty::OutlivesPredicate(ty, _region)) => { ty.visit_with(self) } - ty::PredicateAtom::RegionOutlives(..) => false, + ty::PredicateAtom::RegionOutlives(..) => ControlFlow::CONTINUE, ty::PredicateAtom::ConstEvaluatable(..) if self.def_id_visitor.tcx().features().const_evaluatable_checked => { @@ -103,20 +113,15 @@ where // private function we may have to do something here... // // For now, let's just pretend that everything is fine. - false + ControlFlow::CONTINUE } _ => bug!("unexpected predicate: {:?}", predicate), } } - fn visit_predicates(&mut self, predicates: ty::GenericPredicates<'tcx>) -> bool { + fn visit_predicates(&mut self, predicates: ty::GenericPredicates<'tcx>) -> ControlFlow<()> { let ty::GenericPredicates { parent: _, predicates } = predicates; - for &(predicate, _span) in predicates { - if self.visit_predicate(predicate) { - return true; - } - } - false + predicates.iter().try_for_each(|&(predicate, _span)| self.visit_predicate(predicate)) } } @@ -124,7 +129,7 @@ impl<'tcx, V> TypeVisitor<'tcx> for DefIdVisitorSkeleton<'_, 'tcx, V> where V: DefIdVisitor<'tcx> + ?Sized, { - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { let tcx = self.def_id_visitor.tcx(); // InternalSubsts are not visited here because they are visited below in `super_visit_with`. match *ty.kind() { @@ -133,19 +138,15 @@ where | ty::FnDef(def_id, ..) | ty::Closure(def_id, ..) | ty::Generator(def_id, ..) => { - if self.def_id_visitor.visit_def_id(def_id, "type", &ty) { - return true; - } + self.def_id_visitor.visit_def_id(def_id, "type", &ty)?; if self.def_id_visitor.shallow() { - return false; + return ControlFlow::CONTINUE; } // Default type visitor doesn't visit signatures of fn types. // Something like `fn() -> Priv {my_func}` is considered a private type even if // `my_func` is public, so we need to visit signatures. if let ty::FnDef(..) = ty.kind() { - if tcx.fn_sig(def_id).visit_with(self) { - return true; - } + tcx.fn_sig(def_id).visit_with(self)?; } // Inherent static methods don't have self type in substs. // Something like `fn() {my_method}` type of the method @@ -153,9 +154,7 @@ where // so we need to visit the self type additionally. if let Some(assoc_item) = tcx.opt_associated_item(def_id) { if let ty::ImplContainer(impl_def_id) = assoc_item.container { - if tcx.type_of(impl_def_id).visit_with(self) { - return true; - } + tcx.type_of(impl_def_id).visit_with(self)?; } } } @@ -166,7 +165,7 @@ where // as visible/reachable even if both `Type` and `Trait` are private. // Ideally, associated types should be substituted in the same way as // free type aliases, but this isn't done yet. - return false; + return ControlFlow::CONTINUE; } // This will also visit substs if necessary, so we don't need to recurse. return self.visit_trait(proj.trait_ref(tcx)); @@ -183,9 +182,7 @@ where } }; let ty::ExistentialTraitRef { def_id, substs: _ } = trait_ref; - if self.def_id_visitor.visit_def_id(def_id, "trait", &trait_ref) { - return true; - } + self.def_id_visitor.visit_def_id(def_id, "trait", &trait_ref)?; } } ty::Opaque(def_id, ..) => { @@ -198,12 +195,10 @@ where // through the trait list (default type visitor doesn't visit those traits). // All traits in the list are considered the "primary" part of the type // and are visited by shallow visitors. - if self.visit_predicates(ty::GenericPredicates { + self.visit_predicates(ty::GenericPredicates { parent: None, predicates: tcx.explicit_item_bounds(def_id), - }) { - return true; - } + })?; } } // These types don't have their own def-ids (but may have subcomponents @@ -229,125 +224,10 @@ where } } - !self.def_id_visitor.shallow() && ty.super_visit_with(self) - } -} - -fn def_id_visibility<'tcx>( - tcx: TyCtxt<'tcx>, - def_id: DefId, -) -> (ty::Visibility, Span, &'static str) { - match def_id.as_local().map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id)) { - Some(hir_id) => { - let vis = match tcx.hir().get(hir_id) { - Node::Item(item) => &item.vis, - Node::ForeignItem(foreign_item) => &foreign_item.vis, - Node::MacroDef(macro_def) => { - if tcx.sess.contains_name(¯o_def.attrs, sym::macro_export) { - return (ty::Visibility::Public, macro_def.span, "public"); - } else { - ¯o_def.vis - } - } - Node::TraitItem(..) | Node::Variant(..) => { - return def_id_visibility(tcx, tcx.hir().get_parent_did(hir_id).to_def_id()); - } - Node::ImplItem(impl_item) => { - match tcx.hir().get(tcx.hir().get_parent_item(hir_id)) { - Node::Item(item) => match &item.kind { - hir::ItemKind::Impl { of_trait: None, .. } => &impl_item.vis, - hir::ItemKind::Impl { of_trait: Some(trait_ref), .. } => { - return def_id_visibility(tcx, trait_ref.path.res.def_id()); - } - kind => bug!("unexpected item kind: {:?}", kind), - }, - node => bug!("unexpected node kind: {:?}", node), - } - } - Node::Ctor(vdata) => { - let parent_hir_id = tcx.hir().get_parent_node(hir_id); - match tcx.hir().get(parent_hir_id) { - Node::Variant(..) => { - let parent_did = tcx.hir().local_def_id(parent_hir_id); - let (mut ctor_vis, mut span, mut descr) = - def_id_visibility(tcx, parent_did.to_def_id()); - - let adt_def = tcx.adt_def(tcx.hir().get_parent_did(hir_id).to_def_id()); - let ctor_did = tcx.hir().local_def_id(vdata.ctor_hir_id().unwrap()); - let variant = adt_def.variant_with_ctor_id(ctor_did.to_def_id()); - - if variant.is_field_list_non_exhaustive() - && ctor_vis == ty::Visibility::Public - { - ctor_vis = - ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)); - let attrs = tcx.get_attrs(variant.def_id); - span = tcx - .sess - .find_by_name(&attrs, sym::non_exhaustive) - .unwrap() - .span; - descr = "crate-visible"; - } - - return (ctor_vis, span, descr); - } - Node::Item(..) => { - let item = match tcx.hir().get(parent_hir_id) { - Node::Item(item) => item, - node => bug!("unexpected node kind: {:?}", node), - }; - let (mut ctor_vis, mut span, mut descr) = ( - ty::Visibility::from_hir(&item.vis, parent_hir_id, tcx), - item.vis.span, - item.vis.node.descr(), - ); - for field in vdata.fields() { - let field_vis = ty::Visibility::from_hir(&field.vis, hir_id, tcx); - if ctor_vis.is_at_least(field_vis, tcx) { - ctor_vis = field_vis; - span = field.vis.span; - descr = field.vis.node.descr(); - } - } - - // If the structure is marked as non_exhaustive then lower the - // visibility to within the crate. - if ctor_vis == ty::Visibility::Public { - let adt_def = - tcx.adt_def(tcx.hir().get_parent_did(hir_id).to_def_id()); - if adt_def.non_enum_variant().is_field_list_non_exhaustive() { - ctor_vis = - ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)); - span = tcx - .sess - .find_by_name(&item.attrs, sym::non_exhaustive) - .unwrap() - .span; - descr = "crate-visible"; - } - } - - return (ctor_vis, span, descr); - } - node => bug!("unexpected node kind: {:?}", node), - } - } - Node::Expr(expr) => { - return ( - ty::Visibility::Restricted(tcx.parent_module(expr.hir_id).to_def_id()), - expr.span, - "private", - ); - } - node => bug!("unexpected node kind: {:?}", node), - }; - (ty::Visibility::from_hir(vis, hir_id, tcx), vis.span, vis.node.descr()) - } - None => { - let vis = tcx.visibility(def_id); - let descr = if vis == ty::Visibility::Public { "public" } else { "private" }; - (vis, tcx.def_span(def_id), descr) + if self.def_id_visitor.shallow() { + ControlFlow::CONTINUE + } else { + ty.super_visit_with(self) } } } @@ -398,9 +278,14 @@ impl<'a, 'tcx, VL: VisibilityLike> DefIdVisitor<'tcx> for FindMin<'a, 'tcx, VL> fn skip_assoc_tys(&self) -> bool { true } - fn visit_def_id(&mut self, def_id: DefId, _kind: &str, _descr: &dyn fmt::Display) -> bool { + fn visit_def_id( + &mut self, + def_id: DefId, + _kind: &str, + _descr: &dyn fmt::Display, + ) -> ControlFlow<()> { self.min = VL::new_min(self, def_id); - false + ControlFlow::CONTINUE } } @@ -424,7 +309,7 @@ trait VisibilityLike: Sized { impl VisibilityLike for ty::Visibility { const MAX: Self = ty::Visibility::Public; fn new_min(find: &FindMin<'_, '_, Self>, def_id: DefId) -> Self { - min(def_id_visibility(find.tcx, def_id).0, find.min, find.tcx) + min(find.tcx.visibility(def_id), find.min, find.tcx) } } impl VisibilityLike for Option<AccessLevel> { @@ -534,17 +419,16 @@ impl EmbargoVisitor<'tcx> { let hir_id = item_id.id; let item_def_id = self.tcx.hir().local_def_id(hir_id); let def_kind = self.tcx.def_kind(item_def_id); - let item = self.tcx.hir().expect_item(hir_id); - let vis = ty::Visibility::from_hir(&item.vis, hir_id, self.tcx); + let vis = self.tcx.visibility(item_def_id); self.update_macro_reachable_def(hir_id, def_kind, vis, defining_mod); } if let Some(exports) = self.tcx.module_exports(module_def_id) { for export in exports { if export.vis.is_accessible_from(defining_mod, self.tcx) { if let Res::Def(def_kind, def_id) = export.res { - let vis = def_id_visibility(self.tcx, def_id).0; if let Some(def_id) = def_id.as_local() { let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id); + let vis = self.tcx.visibility(def_id.to_def_id()); self.update_macro_reachable_def(hir_id, def_kind, vis, defining_mod); } } @@ -596,7 +480,7 @@ impl EmbargoVisitor<'tcx> { { for field in struct_def.fields() { let field_vis = - ty::Visibility::from_hir(&field.vis, field.hir_id, self.tcx); + self.tcx.visibility(self.tcx.hir().local_def_id(field.hir_id)); if field_vis.is_accessible_from(module, self.tcx) { self.reach(field.hir_id, level).ty(); } @@ -1013,17 +897,21 @@ impl DefIdVisitor<'tcx> for ReachEverythingInTheInterfaceVisitor<'_, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.ev.tcx } - fn visit_def_id(&mut self, def_id: DefId, _kind: &str, _descr: &dyn fmt::Display) -> bool { + fn visit_def_id( + &mut self, + def_id: DefId, + _kind: &str, + _descr: &dyn fmt::Display, + ) -> ControlFlow<()> { if let Some(def_id) = def_id.as_local() { - let hir_id = self.ev.tcx.hir().local_def_id_to_hir_id(def_id); - if let ((ty::Visibility::Public, ..), _) - | (_, Some(AccessLevel::ReachableFromImplTrait)) = - (def_id_visibility(self.tcx(), def_id.to_def_id()), self.access_level) + if let (ty::Visibility::Public, _) | (_, Some(AccessLevel::ReachableFromImplTrait)) = + (self.tcx().visibility(def_id.to_def_id()), self.access_level) { + let hir_id = self.ev.tcx.hir().local_def_id_to_hir_id(def_id); self.ev.update(hir_id, self.access_level); } } - false + ControlFlow::CONTINUE } } @@ -1184,26 +1072,21 @@ impl<'tcx> TypePrivacyVisitor<'tcx> { } fn item_is_accessible(&self, did: DefId) -> bool { - def_id_visibility(self.tcx, did) - .0 - .is_accessible_from(self.current_item.to_def_id(), self.tcx) + self.tcx.visibility(did).is_accessible_from(self.current_item.to_def_id(), self.tcx) } // Take node-id of an expression or pattern and check its type for privacy. fn check_expr_pat_type(&mut self, id: hir::HirId, span: Span) -> bool { self.span = span; let typeck_results = self.typeck_results(); - if self.visit(typeck_results.node_type(id)) || self.visit(typeck_results.node_substs(id)) { - return true; - } - if let Some(adjustments) = typeck_results.adjustments().get(id) { - for adjustment in adjustments { - if self.visit(adjustment.target) { - return true; - } + let result: ControlFlow<()> = try { + self.visit(typeck_results.node_type(id))?; + self.visit(typeck_results.node_substs(id))?; + if let Some(adjustments) = typeck_results.adjustments().get(id) { + adjustments.iter().try_for_each(|adjustment| self.visit(adjustment.target))?; } - } - false + }; + result.is_break() } fn check_def_id(&mut self, def_id: DefId, kind: &str, descr: &dyn fmt::Display) -> bool { @@ -1245,14 +1128,14 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> { self.span = hir_ty.span; if let Some(typeck_results) = self.maybe_typeck_results { // Types in bodies. - if self.visit(typeck_results.node_type(hir_ty.hir_id)) { + if self.visit(typeck_results.node_type(hir_ty.hir_id)).is_break() { return; } } else { // Types in signatures. // FIXME: This is very ineffective. Ideally each HIR type should be converted // into a semantic type only once and the result should be cached somehow. - if self.visit(rustc_typeck::hir_ty_to_ty(self.tcx, hir_ty)) { + if self.visit(rustc_typeck::hir_ty_to_ty(self.tcx, hir_ty)).is_break() { return; } } @@ -1274,15 +1157,17 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> { ); for (trait_predicate, _, _) in bounds.trait_bounds { - if self.visit_trait(trait_predicate.skip_binder()) { + if self.visit_trait(trait_predicate.skip_binder()).is_break() { return; } } for (poly_predicate, _) in bounds.projection_bounds { let tcx = self.tcx; - if self.visit(poly_predicate.skip_binder().ty) - || self.visit_trait(poly_predicate.skip_binder().projection_ty.trait_ref(tcx)) + if self.visit(poly_predicate.skip_binder().ty).is_break() + || self + .visit_trait(poly_predicate.skip_binder().projection_ty.trait_ref(tcx)) + .is_break() { return; } @@ -1309,7 +1194,7 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> { // Method calls have to be checked specially. self.span = span; if let Some(def_id) = self.typeck_results().type_dependent_def_id(expr.hir_id) { - if self.visit(self.tcx.type_of(def_id)) { + if self.visit(self.tcx.type_of(def_id)).is_break() { return; } } else { @@ -1340,9 +1225,11 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> { .maybe_typeck_results .and_then(|typeck_results| typeck_results.type_dependent_def(id)), }; - let def = def.filter(|(kind, _)| match kind { - DefKind::AssocFn | DefKind::AssocConst | DefKind::AssocTy | DefKind::Static => true, - _ => false, + let def = def.filter(|(kind, _)| { + matches!( + kind, + DefKind::AssocFn | DefKind::AssocConst | DefKind::AssocTy | DefKind::Static + ) }); if let Some((kind, def_id)) = def { let is_local_static = @@ -1407,8 +1294,17 @@ impl DefIdVisitor<'tcx> for TypePrivacyVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } - fn visit_def_id(&mut self, def_id: DefId, kind: &str, descr: &dyn fmt::Display) -> bool { - self.check_def_id(def_id, kind, descr) + fn visit_def_id( + &mut self, + def_id: DefId, + kind: &str, + descr: &dyn fmt::Display, + ) -> ControlFlow<()> { + if self.check_def_id(def_id, kind, descr) { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } } @@ -1840,8 +1736,21 @@ impl SearchInterfaceForPrivateItemsVisitor<'tcx> { None => return false, }; - let (vis, vis_span, vis_descr) = def_id_visibility(self.tcx, def_id); + let vis = self.tcx.visibility(def_id); if !vis.is_at_least(self.required_visibility, self.tcx) { + let vis_descr = match vis { + ty::Visibility::Public => "public", + ty::Visibility::Invisible => "private", + ty::Visibility::Restricted(vis_def_id) => { + if vis_def_id == self.tcx.parent_module(hir_id).to_def_id() { + "private" + } else if vis_def_id.is_top_level_module() { + "crate-private" + } else { + "restricted" + } + } + }; let make_msg = || format!("{} {} `{}` in public interface", vis_descr, kind, descr); if self.has_pub_restricted || self.has_old_errors || self.in_assoc_ty { let mut err = if kind == "trait" { @@ -1849,6 +1758,8 @@ impl SearchInterfaceForPrivateItemsVisitor<'tcx> { } else { struct_span_err!(self.tcx.sess, self.span, E0446, "{}", make_msg()) }; + let vis_span = + self.tcx.sess.source_map().guess_head_span(self.tcx.def_span(def_id)); err.span_label(self.span, format!("can't leak {} {}", vis_descr, kind)); err.span_label(vis_span, format!("`{}` declared as {}", descr, vis_descr)); err.emit(); @@ -1883,8 +1794,17 @@ impl DefIdVisitor<'tcx> for SearchInterfaceForPrivateItemsVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } - fn visit_def_id(&mut self, def_id: DefId, kind: &str, descr: &dyn fmt::Display) -> bool { - self.check_def_id(def_id, kind, descr) + fn visit_def_id( + &mut self, + def_id: DefId, + kind: &str, + descr: &dyn fmt::Display, + ) -> ControlFlow<()> { + if self.check_def_id(def_id, kind, descr) { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } } } @@ -1965,7 +1885,7 @@ impl<'a, 'tcx> Visitor<'tcx> for PrivateItemsInPublicInterfacesVisitor<'a, 'tcx> fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) { let tcx = self.tcx; - let item_visibility = ty::Visibility::from_hir(&item.vis, item.hir_id, tcx); + let item_visibility = tcx.visibility(tcx.hir().local_def_id(item.hir_id).to_def_id()); match item.kind { // Crates are always public. @@ -2019,7 +1939,7 @@ impl<'a, 'tcx> Visitor<'tcx> for PrivateItemsInPublicInterfacesVisitor<'a, 'tcx> // Subitems of foreign modules have their own publicity. hir::ItemKind::ForeignMod(ref foreign_mod) => { for foreign_item in foreign_mod.items { - let vis = ty::Visibility::from_hir(&foreign_item.vis, item.hir_id, tcx); + let vis = tcx.visibility(tcx.hir().local_def_id(foreign_item.hir_id)); self.check(foreign_item.hir_id, vis).generics().predicates().ty(); } } @@ -2028,7 +1948,7 @@ impl<'a, 'tcx> Visitor<'tcx> for PrivateItemsInPublicInterfacesVisitor<'a, 'tcx> self.check(item.hir_id, item_visibility).generics().predicates(); for field in struct_def.fields() { - let field_visibility = ty::Visibility::from_hir(&field.vis, item.hir_id, tcx); + let field_visibility = tcx.visibility(tcx.hir().local_def_id(field.hir_id)); self.check(field.hir_id, min(item_visibility, field_visibility, tcx)).ty(); } } @@ -2040,10 +1960,9 @@ impl<'a, 'tcx> Visitor<'tcx> for PrivateItemsInPublicInterfacesVisitor<'a, 'tcx> let impl_vis = ty::Visibility::of_impl(item.hir_id, tcx, &Default::default()); self.check(item.hir_id, impl_vis).generics().predicates(); for impl_item_ref in items { - let impl_item = tcx.hir().impl_item(impl_item_ref.id); let impl_item_vis = if of_trait.is_none() { min( - ty::Visibility::from_hir(&impl_item.vis, item.hir_id, tcx), + tcx.visibility(tcx.hir().local_def_id(impl_item_ref.id.hir_id)), impl_vis, tcx, ) @@ -2064,6 +1983,7 @@ impl<'a, 'tcx> Visitor<'tcx> for PrivateItemsInPublicInterfacesVisitor<'a, 'tcx> pub fn provide(providers: &mut Providers) { *providers = Providers { + visibility, privacy_access_levels, check_private_in_public, check_mod_privacy, @@ -2071,6 +1991,55 @@ pub fn provide(providers: &mut Providers) { }; } +fn visibility(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Visibility { + let def_id = def_id.expect_local(); + match tcx.visibilities.get(&def_id) { + Some(vis) => *vis, + None => { + let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); + match tcx.hir().get(hir_id) { + // Unique types created for closures participate in type privacy checking. + // They have visibilities inherited from the module they are defined in. + Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(..), .. }) => { + ty::Visibility::Restricted(tcx.parent_module(hir_id).to_def_id()) + } + // - AST lowering may clone `use` items and the clones don't + // get their entries in the resolver's visibility table. + // - AST lowering also creates opaque type items with inherited visibilies. + // Visibility on them should have no effect, but to avoid the visibility + // query failing on some items, we provide it for opaque types as well. + Node::Item(hir::Item { + vis, + kind: hir::ItemKind::Use(..) | hir::ItemKind::OpaqueTy(..), + .. + }) => ty::Visibility::from_hir(vis, hir_id, tcx), + // Visibilities of trait impl items are inherited from their traits + // and are not filled in resolve. + Node::ImplItem(impl_item) => { + match tcx.hir().get(tcx.hir().get_parent_item(hir_id)) { + Node::Item(hir::Item { + kind: hir::ItemKind::Impl { of_trait: Some(tr), .. }, + .. + }) => tr.path.res.opt_def_id().map_or_else( + || { + tcx.sess.delay_span_bug(tr.path.span, "trait without a def-id"); + ty::Visibility::Public + }, + |def_id| tcx.visibility(def_id), + ), + _ => span_bug!(impl_item.span, "the parent is not a trait impl"), + } + } + _ => span_bug!( + tcx.def_span(def_id), + "visibility table unexpectedly missing a def-id: {:?}", + def_id, + ), + } + } + } +} + fn check_mod_privacy(tcx: TyCtxt<'_>, module_def_id: LocalDefId) { // Check privacy of names not checked in previous compilation stages. let mut visitor = NamePrivacyVisitor { tcx, maybe_typeck_results: None, current_item: None }; diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 85335f0ba50..d9b687c48af 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -292,10 +292,8 @@ impl<K: DepKind> DepGraph<K> { ); data.colors.insert(prev_index, color); - } else { - if print_status { - eprintln!("[task::new] {:?}", key); - } + } else if print_status { + eprintln!("[task::new] {:?}", key); } (result, dep_node_index) diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 1839e1af45e..7bc6ae1d1c6 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -1,12 +1,12 @@ use crate::dep_graph::DepNodeIndex; use crate::query::plumbing::{QueryLookup, QueryState}; -use crate::query::QueryContext; use rustc_arena::TypedArena; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::WorkerLocal; use std::default::Default; +use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; @@ -24,16 +24,16 @@ pub trait QueryStorage: Default { } pub trait QueryCache: QueryStorage { - type Key: Hash; + type Key: Hash + Eq + Clone + Debug; type Sharded: Default; /// Checks if the query is already computed and in the cache. /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need /// to compute it. - fn lookup<CTX: QueryContext, R, OnHit, OnMiss>( + fn lookup<D, Q, R, OnHit, OnMiss>( &self, - state: &QueryState<CTX, Self>, + state: &QueryState<D, Q, Self>, key: Self::Key, // `on_hit` can be called while holding a lock to the query state shard. on_hit: OnHit, @@ -41,7 +41,7 @@ pub trait QueryCache: QueryStorage { ) -> R where OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R, - OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R; + OnMiss: FnOnce(Self::Key, QueryLookup<'_, D, Q, Self::Key, Self::Sharded>) -> R; fn complete( &self, @@ -86,21 +86,25 @@ impl<K: Eq + Hash, V: Clone> QueryStorage for DefaultCache<K, V> { } } -impl<K: Eq + Hash, V: Clone> QueryCache for DefaultCache<K, V> { +impl<K, V> QueryCache for DefaultCache<K, V> +where + K: Eq + Hash + Clone + Debug, + V: Clone, +{ type Key = K; type Sharded = FxHashMap<K, (V, DepNodeIndex)>; #[inline(always)] - fn lookup<CTX: QueryContext, R, OnHit, OnMiss>( + fn lookup<D, Q, R, OnHit, OnMiss>( &self, - state: &QueryState<CTX, Self>, + state: &QueryState<D, Q, Self>, key: K, on_hit: OnHit, on_miss: OnMiss, ) -> R where OnHit: FnOnce(&V, DepNodeIndex) -> R, - OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, + OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R, { let mut lookup = state.get_lookup(&key); let lock = &mut *lookup.lock; @@ -164,21 +168,24 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> QueryStorage for ArenaCache<'tcx, K, V> { } } -impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { +impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> +where + K: Eq + Hash + Clone + Debug, +{ type Key = K; type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>; #[inline(always)] - fn lookup<CTX: QueryContext, R, OnHit, OnMiss>( + fn lookup<D, Q, R, OnHit, OnMiss>( &self, - state: &QueryState<CTX, Self>, + state: &QueryState<D, Q, Self>, key: K, on_hit: OnHit, on_miss: OnMiss, ) -> R where OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R, - OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, + OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R, { let mut lookup = state.get_lookup(&key); let lock = &mut *lookup.lock; diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 549056570f9..0f0684b3547 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -5,18 +5,14 @@ use crate::dep_graph::SerializedDepNodeIndex; use crate::query::caches::QueryCache; use crate::query::plumbing::CycleError; use crate::query::{QueryContext, QueryState}; -use rustc_data_structures::profiling::ProfileCategory; use rustc_data_structures::fingerprint::Fingerprint; use std::borrow::Cow; use std::fmt::Debug; use std::hash::Hash; -// The parameter `CTX` is required in librustc_middle: -// implementations may need to access the `'tcx` lifetime in `CTX = TyCtxt<'tcx>`. -pub trait QueryConfig<CTX> { +pub trait QueryConfig { const NAME: &'static str; - const CATEGORY: ProfileCategory; type Key: Eq + Hash + Clone + Debug; type Value; @@ -70,7 +66,7 @@ impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> { } } -pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> { +pub trait QueryAccessors<CTX: QueryContext>: QueryConfig { const ANON: bool; const EVAL_ALWAYS: bool; const DEP_KIND: CTX::DepKind; @@ -78,7 +74,7 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> { type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>; + fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX::DepKind, CTX::Query, Self::Cache>; fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind> where diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 190312bb330..c1d3210b617 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -1,16 +1,16 @@ -use crate::dep_graph::{DepContext, DepKind}; use crate::query::plumbing::CycleError; -use crate::query::QueryContext; use rustc_data_structures::fx::FxHashMap; use rustc_span::Span; use std::convert::TryFrom; +use std::hash::Hash; use std::marker::PhantomData; use std::num::NonZeroU32; #[cfg(parallel_compiler)] use { + super::QueryContext, parking_lot::{Condvar, Mutex}, rustc_data_structures::fx::FxHashSet, rustc_data_structures::stable_hasher::{HashStable, StableHasher}, @@ -31,7 +31,7 @@ pub struct QueryInfo<Q> { pub query: Q, } -type QueryMap<CTX> = FxHashMap<QueryJobId<<CTX as DepContext>::DepKind>, QueryJobInfo<CTX>>; +pub(crate) type QueryMap<D, Q> = FxHashMap<QueryJobId<D>, QueryJobInfo<D, Q>>; /// A value uniquely identifiying an active query job within a shard in the query cache. #[derive(Copy, Clone, Eq, PartialEq, Hash)] @@ -39,71 +39,75 @@ pub struct QueryShardJobId(pub NonZeroU32); /// A value uniquely identifiying an active query job. #[derive(Copy, Clone, Eq, PartialEq, Hash)] -pub struct QueryJobId<K> { +pub struct QueryJobId<D> { /// Which job within a shard is this pub job: QueryShardJobId, /// In which shard is this job pub shard: u16, - /// What kind of query this job is - pub kind: K, + /// What kind of query this job is. + pub kind: D, } -impl<K: DepKind> QueryJobId<K> { - pub fn new(job: QueryShardJobId, shard: usize, kind: K) -> Self { +impl<D> QueryJobId<D> +where + D: Copy + Clone + Eq + Hash, +{ + pub fn new(job: QueryShardJobId, shard: usize, kind: D) -> Self { QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind } } - fn query<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> CTX::Query { + fn query<Q: Clone>(self, map: &QueryMap<D, Q>) -> Q { map.get(&self).unwrap().info.query.clone() } #[cfg(parallel_compiler)] - fn span<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Span { + fn span<Q: Clone>(self, map: &QueryMap<D, Q>) -> Span { map.get(&self).unwrap().job.span } #[cfg(parallel_compiler)] - fn parent<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Option<QueryJobId<K>> { + fn parent<Q: Clone>(self, map: &QueryMap<D, Q>) -> Option<QueryJobId<D>> { map.get(&self).unwrap().job.parent } #[cfg(parallel_compiler)] - fn latch<'a, CTX: QueryContext<DepKind = K>>( - self, - map: &'a QueryMap<CTX>, - ) -> Option<&'a QueryLatch<CTX>> { + fn latch<'a, Q: Clone>(self, map: &'a QueryMap<D, Q>) -> Option<&'a QueryLatch<D, Q>> { map.get(&self).unwrap().job.latch.as_ref() } } -pub struct QueryJobInfo<CTX: QueryContext> { - pub info: QueryInfo<CTX::Query>, - pub job: QueryJob<CTX>, +pub struct QueryJobInfo<D, Q> { + pub info: QueryInfo<Q>, + pub job: QueryJob<D, Q>, } /// Represents an active query job. #[derive(Clone)] -pub struct QueryJob<CTX: QueryContext> { +pub struct QueryJob<D, Q> { pub id: QueryShardJobId, /// The span corresponding to the reason for which this query was required. pub span: Span, /// The parent query job which created this job and is implicitly waiting on it. - pub parent: Option<QueryJobId<CTX::DepKind>>, + pub parent: Option<QueryJobId<D>>, /// The latch that is used to wait on this job. #[cfg(parallel_compiler)] - latch: Option<QueryLatch<CTX>>, + latch: Option<QueryLatch<D, Q>>, - dummy: PhantomData<QueryLatch<CTX>>, + dummy: PhantomData<QueryLatch<D, Q>>, } -impl<CTX: QueryContext> QueryJob<CTX> { +impl<D, Q> QueryJob<D, Q> +where + D: Copy + Clone + Eq + Hash, + Q: Clone, +{ /// Creates a new query job. - pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<CTX::DepKind>>) -> Self { + pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<D>>) -> Self { QueryJob { id, span, @@ -115,7 +119,7 @@ impl<CTX: QueryContext> QueryJob<CTX> { } #[cfg(parallel_compiler)] - pub(super) fn latch(&mut self, _id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> { + pub(super) fn latch(&mut self, _id: QueryJobId<D>) -> QueryLatch<D, Q> { if self.latch.is_none() { self.latch = Some(QueryLatch::new()); } @@ -123,7 +127,7 @@ impl<CTX: QueryContext> QueryJob<CTX> { } #[cfg(not(parallel_compiler))] - pub(super) fn latch(&mut self, id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> { + pub(super) fn latch(&mut self, id: QueryJobId<D>) -> QueryLatch<D, Q> { QueryLatch { id, dummy: PhantomData } } @@ -143,19 +147,26 @@ impl<CTX: QueryContext> QueryJob<CTX> { #[cfg(not(parallel_compiler))] #[derive(Clone)] -pub(super) struct QueryLatch<CTX: QueryContext> { - id: QueryJobId<CTX::DepKind>, - dummy: PhantomData<CTX>, +pub(super) struct QueryLatch<D, Q> { + id: QueryJobId<D>, + dummy: PhantomData<Q>, } #[cfg(not(parallel_compiler))] -impl<CTX: QueryContext> QueryLatch<CTX> { - pub(super) fn find_cycle_in_stack(&self, tcx: CTX, span: Span) -> CycleError<CTX::Query> { - let query_map = tcx.try_collect_active_jobs().unwrap(); - - // Get the current executing query (waiter) and find the waitee amongst its parents - let mut current_job = tcx.current_query_job(); +impl<D, Q> QueryLatch<D, Q> +where + D: Copy + Clone + Eq + Hash, + Q: Clone, +{ + pub(super) fn find_cycle_in_stack( + &self, + query_map: QueryMap<D, Q>, + current_job: &Option<QueryJobId<D>>, + span: Span, + ) -> CycleError<Q> { + // Find the waitee amongst `current_job` parents let mut cycle = Vec::new(); + let mut current_job = Option::clone(current_job); while let Some(job) = current_job { let info = query_map.get(&job).unwrap(); @@ -186,15 +197,15 @@ impl<CTX: QueryContext> QueryLatch<CTX> { } #[cfg(parallel_compiler)] -struct QueryWaiter<CTX: QueryContext> { - query: Option<QueryJobId<CTX::DepKind>>, +struct QueryWaiter<D, Q> { + query: Option<QueryJobId<D>>, condvar: Condvar, span: Span, - cycle: Lock<Option<CycleError<CTX::Query>>>, + cycle: Lock<Option<CycleError<Q>>>, } #[cfg(parallel_compiler)] -impl<CTX: QueryContext> QueryWaiter<CTX> { +impl<D, Q> QueryWaiter<D, Q> { fn notify(&self, registry: &rayon_core::Registry) { rayon_core::mark_unblocked(registry); self.condvar.notify_one(); @@ -202,19 +213,19 @@ impl<CTX: QueryContext> QueryWaiter<CTX> { } #[cfg(parallel_compiler)] -struct QueryLatchInfo<CTX: QueryContext> { +struct QueryLatchInfo<D, Q> { complete: bool, - waiters: Vec<Lrc<QueryWaiter<CTX>>>, + waiters: Vec<Lrc<QueryWaiter<D, Q>>>, } #[cfg(parallel_compiler)] #[derive(Clone)] -pub(super) struct QueryLatch<CTX: QueryContext> { - info: Lrc<Mutex<QueryLatchInfo<CTX>>>, +pub(super) struct QueryLatch<D, Q> { + info: Lrc<Mutex<QueryLatchInfo<D, Q>>>, } #[cfg(parallel_compiler)] -impl<CTX: QueryContext> QueryLatch<CTX> { +impl<D: Eq + Hash, Q: Clone> QueryLatch<D, Q> { fn new() -> Self { QueryLatch { info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })), @@ -223,10 +234,13 @@ impl<CTX: QueryContext> QueryLatch<CTX> { } #[cfg(parallel_compiler)] -impl<CTX: QueryContext> QueryLatch<CTX> { +impl<D, Q> QueryLatch<D, Q> { /// Awaits for the query job to complete. - pub(super) fn wait_on(&self, tcx: CTX, span: Span) -> Result<(), CycleError<CTX::Query>> { - let query = tcx.current_query_job(); + pub(super) fn wait_on( + &self, + query: Option<QueryJobId<D>>, + span: Span, + ) -> Result<(), CycleError<Q>> { let waiter = Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() }); self.wait_on_inner(&waiter); @@ -239,12 +253,9 @@ impl<CTX: QueryContext> QueryLatch<CTX> { Some(cycle) => Err(cycle), } } -} -#[cfg(parallel_compiler)] -impl<CTX: QueryContext> QueryLatch<CTX> { /// Awaits the caller on this latch by blocking the current thread. - fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<CTX>>) { + fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<D, Q>>) { let mut info = self.info.lock(); if !info.complete { // We push the waiter on to the `waiters` list. It can be accessed inside @@ -278,7 +289,7 @@ impl<CTX: QueryContext> QueryLatch<CTX> { /// Removes a single waiter from the list of waiters. /// This is used to break query cycles. - fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<CTX>> { + fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<D, Q>> { let mut info = self.info.lock(); debug_assert!(!info.complete); // Remove the waiter from the list of waiters @@ -288,7 +299,7 @@ impl<CTX: QueryContext> QueryLatch<CTX> { /// A resumable waiter of a query. The usize is the index into waiters in the query's latch #[cfg(parallel_compiler)] -type Waiter<K> = (QueryJobId<K>, usize); +type Waiter<D> = (QueryJobId<D>, usize); /// Visits all the non-resumable and resumable waiters of a query. /// Only waiters in a query are visited. @@ -300,13 +311,15 @@ type Waiter<K> = (QueryJobId<K>, usize); /// required information to resume the waiter. /// If all `visit` calls returns None, this function also returns None. #[cfg(parallel_compiler)] -fn visit_waiters<CTX: QueryContext, F>( - query_map: &QueryMap<CTX>, - query: QueryJobId<CTX::DepKind>, +fn visit_waiters<D, Q, F>( + query_map: &QueryMap<D, Q>, + query: QueryJobId<D>, mut visit: F, -) -> Option<Option<Waiter<CTX::DepKind>>> +) -> Option<Option<Waiter<D>>> where - F: FnMut(Span, QueryJobId<CTX::DepKind>) -> Option<Option<Waiter<CTX::DepKind>>>, + D: Copy + Clone + Eq + Hash, + Q: Clone, + F: FnMut(Span, QueryJobId<D>) -> Option<Option<Waiter<D>>>, { // Visit the parent query which is a non-resumable waiter since it's on the same stack if let Some(parent) = query.parent(query_map) { @@ -335,13 +348,17 @@ where /// If a cycle is detected, this initial value is replaced with the span causing /// the cycle. #[cfg(parallel_compiler)] -fn cycle_check<CTX: QueryContext>( - query_map: &QueryMap<CTX>, - query: QueryJobId<CTX::DepKind>, +fn cycle_check<D, Q>( + query_map: &QueryMap<D, Q>, + query: QueryJobId<D>, span: Span, - stack: &mut Vec<(Span, QueryJobId<CTX::DepKind>)>, - visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>, -) -> Option<Option<Waiter<CTX::DepKind>>> { + stack: &mut Vec<(Span, QueryJobId<D>)>, + visited: &mut FxHashSet<QueryJobId<D>>, +) -> Option<Option<Waiter<D>>> +where + D: Copy + Clone + Eq + Hash, + Q: Clone, +{ if !visited.insert(query) { return if let Some(p) = stack.iter().position(|q| q.1 == query) { // We detected a query cycle, fix up the initial span and return Some @@ -376,11 +393,15 @@ fn cycle_check<CTX: QueryContext>( /// from `query` without going through any of the queries in `visited`. /// This is achieved with a depth first search. #[cfg(parallel_compiler)] -fn connected_to_root<CTX: QueryContext>( - query_map: &QueryMap<CTX>, - query: QueryJobId<CTX::DepKind>, - visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>, -) -> bool { +fn connected_to_root<D, Q>( + query_map: &QueryMap<D, Q>, + query: QueryJobId<D>, + visited: &mut FxHashSet<QueryJobId<D>>, +) -> bool +where + D: Copy + Clone + Eq + Hash, + Q: Clone, +{ // We already visited this or we're deliberately ignoring it if !visited.insert(query) { return false; @@ -399,7 +420,12 @@ fn connected_to_root<CTX: QueryContext>( // Deterministically pick an query from a list #[cfg(parallel_compiler)] -fn pick_query<'a, CTX, T, F>(query_map: &QueryMap<CTX>, tcx: CTX, queries: &'a [T], f: F) -> &'a T +fn pick_query<'a, CTX, T, F>( + query_map: &QueryMap<CTX::DepKind, CTX::Query>, + tcx: CTX, + queries: &'a [T], + f: F, +) -> &'a T where CTX: QueryContext, F: Fn(&T) -> (Span, QueryJobId<CTX::DepKind>), @@ -429,9 +455,9 @@ where /// the function returns false. #[cfg(parallel_compiler)] fn remove_cycle<CTX: QueryContext>( - query_map: &QueryMap<CTX>, + query_map: &QueryMap<CTX::DepKind, CTX::Query>, jobs: &mut Vec<QueryJobId<CTX::DepKind>>, - wakelist: &mut Vec<Lrc<QueryWaiter<CTX>>>, + wakelist: &mut Vec<Lrc<QueryWaiter<CTX::DepKind, CTX::Query>>>, tcx: CTX, ) -> bool { let mut visited = FxHashSet::default(); diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index 49097725bc9..da45565dbe6 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -15,8 +15,8 @@ mod config; pub use self::config::{QueryAccessors, QueryConfig, QueryDescription}; use crate::dep_graph::{DepContext, DepGraph}; +use crate::query::job::QueryMap; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; @@ -38,9 +38,7 @@ pub trait QueryContext: DepContext { /// Get the query information from the TLS context. fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>>; - fn try_collect_active_jobs( - &self, - ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>>; + fn try_collect_active_jobs(&self) -> Option<QueryMap<Self::DepKind, Self::Query>>; /// Executes a job by changing the `ImplicitCtxt` to point to the /// new query job while it executes. It returns the diagnostics diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index ae042cc8081..426f5bb41d6 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -7,7 +7,7 @@ use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId}; -use crate::query::QueryContext; +use crate::query::{QueryContext, QueryMap}; #[cfg(not(parallel_compiler))] use rustc_data_structures::cold_path; @@ -20,8 +20,6 @@ use rustc_errors::{Diagnostic, FatalError}; use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; use std::collections::hash_map::Entry; -use std::convert::TryFrom; -use std::fmt::Debug; use std::hash::{Hash, Hasher}; use std::mem; use std::num::NonZeroU32; @@ -29,33 +27,33 @@ use std::ptr; #[cfg(debug_assertions)] use std::sync::atomic::{AtomicUsize, Ordering}; -pub(super) struct QueryStateShard<CTX: QueryContext, K, C> { +pub(super) struct QueryStateShard<D, Q, K, C> { pub(super) cache: C, - active: FxHashMap<K, QueryResult<CTX>>, + active: FxHashMap<K, QueryResult<D, Q>>, /// Used to generate unique ids for active jobs. jobs: u32, } -impl<CTX: QueryContext, K, C: Default> Default for QueryStateShard<CTX, K, C> { - fn default() -> QueryStateShard<CTX, K, C> { +impl<D, Q, K, C: Default> Default for QueryStateShard<D, Q, K, C> { + fn default() -> QueryStateShard<D, Q, K, C> { QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 } } } -pub struct QueryState<CTX: QueryContext, C: QueryCache> { +pub struct QueryState<D, Q, C: QueryCache> { cache: C, - shards: Sharded<QueryStateShard<CTX, C::Key, C::Sharded>>, + shards: Sharded<QueryStateShard<D, Q, C::Key, C::Sharded>>, #[cfg(debug_assertions)] pub cache_hits: AtomicUsize, } -impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { +impl<D, Q, C: QueryCache> QueryState<D, Q, C> { #[inline] pub(super) fn get_lookup<'tcx>( &'tcx self, key: &C::Key, - ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> { + ) -> QueryLookup<'tcx, D, Q, C::Key, C::Sharded> { // We compute the key's hash once and then use it for both the // shard lookup and the hashmap lookup. This relies on the fact // that both of them use `FxHasher`. @@ -70,16 +68,21 @@ impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { } /// Indicates the state of a query for a given key in a query map. -enum QueryResult<CTX: QueryContext> { +enum QueryResult<D, Q> { /// An already executing query. The query job can be used to await for its completion. - Started(QueryJob<CTX>), + Started(QueryJob<D, Q>), /// The query panicked. Queries trying to wait on this will raise a fatal error which will /// silently panic. Poisoned, } -impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { +impl<D, Q, C> QueryState<D, Q, C> +where + D: Copy + Clone + Eq + Hash, + Q: Clone, + C: QueryCache, +{ #[inline(always)] pub fn iter_results<R>( &self, @@ -98,13 +101,10 @@ impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { pub fn try_collect_active_jobs( &self, - kind: CTX::DepKind, - make_query: fn(C::Key) -> CTX::Query, - jobs: &mut FxHashMap<QueryJobId<CTX::DepKind>, QueryJobInfo<CTX>>, - ) -> Option<()> - where - C::Key: Clone, - { + kind: D, + make_query: fn(C::Key) -> Q, + jobs: &mut QueryMap<D, Q>, + ) -> Option<()> { // We use try_lock_shards here since we are called from the // deadlock handler, and this shouldn't be locked. let shards = self.shards.try_lock_shards()?; @@ -112,8 +112,7 @@ impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { jobs.extend(shards.flat_map(|(shard_id, shard)| { shard.active.iter().filter_map(move |(k, v)| { if let QueryResult::Started(ref job) = *v { - let id = - QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind }; + let id = QueryJobId::new(job.id, shard_id, kind); let info = QueryInfo { span: job.span, query: make_query(k.clone()) }; Some((id, QueryJobInfo { info, job: job.clone() })) } else { @@ -126,8 +125,8 @@ impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> { } } -impl<CTX: QueryContext, C: QueryCache> Default for QueryState<CTX, C> { - fn default() -> QueryState<CTX, C> { +impl<D, Q, C: QueryCache> Default for QueryState<D, Q, C> { + fn default() -> QueryState<D, Q, C> { QueryState { cache: C::default(), shards: Default::default(), @@ -138,28 +137,30 @@ impl<CTX: QueryContext, C: QueryCache> Default for QueryState<CTX, C> { } /// Values used when checking a query cache which can be reused on a cache-miss to execute the query. -pub struct QueryLookup<'tcx, CTX: QueryContext, K, C> { +pub struct QueryLookup<'tcx, D, Q, K, C> { pub(super) key_hash: u64, shard: usize, - pub(super) lock: LockGuard<'tcx, QueryStateShard<CTX, K, C>>, + pub(super) lock: LockGuard<'tcx, QueryStateShard<D, Q, K, C>>, } /// A type representing the responsibility to execute the job in the `job` field. /// This will poison the relevant query if dropped. -struct JobOwner<'tcx, CTX: QueryContext, C> +struct JobOwner<'tcx, D, Q, C> where + D: Copy + Clone + Eq + Hash, + Q: Clone, C: QueryCache, - C::Key: Eq + Hash + Clone + Debug, { - state: &'tcx QueryState<CTX, C>, + state: &'tcx QueryState<D, Q, C>, key: C::Key, - id: QueryJobId<CTX::DepKind>, + id: QueryJobId<D>, } -impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C> +impl<'tcx, D, Q, C> JobOwner<'tcx, D, Q, C> where + D: Copy + Clone + Eq + Hash, + Q: Clone, C: QueryCache, - C::Key: Eq + Hash + Clone + Debug, { /// Either gets a `JobOwner` corresponding the query, allowing us to /// start executing the query, or returns with the result of the query. @@ -170,14 +171,14 @@ where /// This function is inlined because that results in a noticeable speed-up /// for some compile-time benchmarks. #[inline(always)] - fn try_start<'a, 'b>( + fn try_start<'a, 'b, CTX>( tcx: CTX, - state: &'b QueryState<CTX, C>, + state: &'b QueryState<CTX::DepKind, CTX::Query, C>, span: Span, key: &C::Key, - mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>, + mut lookup: QueryLookup<'a, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, query: &QueryVtable<CTX, C::Key, C::Value>, - ) -> TryGetJob<'b, CTX, C> + ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C> where CTX: QueryContext, { @@ -229,7 +230,12 @@ where // so we just return the error. #[cfg(not(parallel_compiler))] return TryGetJob::Cycle(cold_path(|| { - let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)); + let error: CycleError<CTX::Query> = latch.find_cycle_in_stack( + tcx.try_collect_active_jobs().unwrap(), + &tcx.current_query_job(), + span, + ); + let value = query.handle_cycle_error(tcx, error); state.cache.store_nocache(value) })); @@ -237,7 +243,7 @@ where // thread. #[cfg(parallel_compiler)] { - let result = latch.wait_on(tcx, span); + let result = latch.wait_on(tcx.current_query_job(), span); if let Err(cycle) = result { let value = query.handle_cycle_error(tcx, cycle); @@ -297,9 +303,11 @@ where (result, diagnostics.into_inner()) } -impl<'tcx, CTX: QueryContext, C: QueryCache> Drop for JobOwner<'tcx, CTX, C> +impl<'tcx, D, Q, C> Drop for JobOwner<'tcx, D, Q, C> where - C::Key: Eq + Hash + Clone + Debug, + D: Copy + Clone + Eq + Hash, + Q: Clone, + C: QueryCache, { #[inline(never)] #[cold] @@ -330,12 +338,14 @@ pub struct CycleError<Q> { } /// The result of `try_start`. -enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache> +enum TryGetJob<'tcx, D, Q, C> where - C::Key: Eq + Hash + Clone + Debug, + D: Copy + Clone + Eq + Hash, + Q: Clone, + C: QueryCache, { /// The query is not yet started. Contains a guard to the cache eventually used to start it. - NotYetStarted(JobOwner<'tcx, CTX, C>), + NotYetStarted(JobOwner<'tcx, D, Q, C>), /// The query was already completed. /// Returns the result of the query and its dep-node index @@ -354,7 +364,7 @@ where #[inline(always)] fn try_get_cached<CTX, C, R, OnHit, OnMiss>( tcx: CTX, - state: &QueryState<CTX, C>, + state: &QueryState<CTX::DepKind, CTX::Query, C>, key: C::Key, // `on_hit` can be called while holding a lock to the query cache on_hit: OnHit, @@ -364,7 +374,7 @@ where C: QueryCache, CTX: QueryContext, OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R, - OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R, + OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>) -> R, { state.cache.lookup( state, @@ -386,19 +396,20 @@ where #[inline(always)] fn try_execute_query<CTX, C>( tcx: CTX, - state: &QueryState<CTX, C>, + state: &QueryState<CTX::DepKind, CTX::Query, C>, span: Span, key: C::Key, - lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>, + lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, query: &QueryVtable<CTX, C::Key, C::Value>, ) -> C::Stored where C: QueryCache, - C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams<CTX>, - C::Stored: Clone, + C::Key: crate::dep_graph::DepNodeParams<CTX>, CTX: QueryContext, { - let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { + let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( + tcx, state, span, &key, lookup, query, + ) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, #[cfg(parallel_compiler)] @@ -559,14 +570,12 @@ fn incremental_verify_ich<CTX, K, V>( fn force_query_with_job<C, CTX>( tcx: CTX, key: C::Key, - job: JobOwner<'_, CTX, C>, + job: JobOwner<'_, CTX::DepKind, CTX::Query, C>, dep_node: DepNode<CTX::DepKind>, query: &QueryVtable<CTX, C::Key, C::Value>, ) -> (C::Stored, DepNodeIndex) where C: QueryCache, - C::Key: Eq + Clone + Debug, - C::Stored: Clone, CTX: QueryContext, { // If the following assertion triggers, it can have two reasons: @@ -603,10 +612,8 @@ where prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - if unlikely!(!diagnostics.is_empty()) { - if dep_node.kind != DepKind::NULL { - tcx.store_diagnostics(dep_node_index, diagnostics); - } + if unlikely!(!diagnostics.is_empty()) && dep_node.kind != DepKind::NULL { + tcx.store_diagnostics(dep_node_index, diagnostics); } let result = job.complete(result, dep_node_index); @@ -617,7 +624,7 @@ where #[inline(never)] fn get_query_impl<CTX, C>( tcx: CTX, - state: &QueryState<CTX, C>, + state: &QueryState<CTX::DepKind, CTX::Query, C>, span: Span, key: C::Key, query: &QueryVtable<CTX, C::Key, C::Value>, @@ -625,8 +632,7 @@ fn get_query_impl<CTX, C>( where CTX: QueryContext, C: QueryCache, - C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>, - C::Stored: Clone, + C::Key: crate::dep_graph::DepNodeParams<CTX>, { try_get_cached( tcx, @@ -650,12 +656,12 @@ where #[inline(never)] fn ensure_query_impl<CTX, C>( tcx: CTX, - state: &QueryState<CTX, C>, + state: &QueryState<CTX::DepKind, CTX::Query, C>, key: C::Key, query: &QueryVtable<CTX, C::Key, C::Value>, ) where C: QueryCache, - C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>, + C::Key: crate::dep_graph::DepNodeParams<CTX>, CTX: QueryContext, { if query.eval_always { @@ -687,14 +693,14 @@ fn ensure_query_impl<CTX, C>( #[inline(never)] fn force_query_impl<CTX, C>( tcx: CTX, - state: &QueryState<CTX, C>, + state: &QueryState<CTX::DepKind, CTX::Query, C>, key: C::Key, span: Span, dep_node: DepNode<CTX::DepKind>, query: &QueryVtable<CTX, C::Key, C::Value>, ) where C: QueryCache, - C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>, + C::Key: crate::dep_graph::DepNodeParams<CTX>, CTX: QueryContext, { // We may be concurrently trying both execute and force a query. @@ -708,7 +714,9 @@ fn force_query_impl<CTX, C>( // Cache hit, do nothing }, |key, lookup| { - let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { + let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( + tcx, state, span, &key, lookup, query, + ) { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(_) => return, #[cfg(parallel_compiler)] diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs index a48d002b2a3..d2d2a79374e 100644 --- a/compiler/rustc_resolve/src/build_reduced_graph.rs +++ b/compiler/rustc_resolve/src/build_reduced_graph.rs @@ -95,6 +95,27 @@ impl<'a> Resolver<'a> { } } + /// Walks up the tree of definitions starting at `def_id`, + /// stopping at the first `DefKind::Mod` encountered + fn nearest_mod_parent(&mut self, def_id: DefId) -> Module<'a> { + let def_key = self.cstore().def_key(def_id); + + let mut parent_id = DefId { + krate: def_id.krate, + index: def_key.parent.expect("failed to get parent for module"), + }; + // The immediate parent may not be a module + // (e.g. `const _: () = { #[path = "foo.rs"] mod foo; };`) + // Walk up the tree until we hit a module or the crate root. + while parent_id.index != CRATE_DEF_INDEX + && self.cstore().def_kind(parent_id) != DefKind::Mod + { + let parent_def_key = self.cstore().def_key(parent_id); + parent_id.index = parent_def_key.parent.expect("failed to get parent for module"); + } + self.get_module(parent_id) + } + crate fn get_module(&mut self, def_id: DefId) -> Module<'a> { // If this is a local module, it will be in `module_map`, no need to recalculate it. if let Some(def_id) = def_id.as_local() { @@ -116,11 +137,8 @@ impl<'a> Resolver<'a> { .data .get_opt_name() .expect("given a DefId that wasn't a module"); - // This unwrap is safe since we know this isn't the root - let parent = Some(self.get_module(DefId { - index: def_key.parent.expect("failed to get parent for module"), - ..def_id - })); + + let parent = Some(self.nearest_mod_parent(def_id)); (name, parent) }; @@ -145,8 +163,24 @@ impl<'a> Resolver<'a> { if let Some(id) = def_id.as_local() { self.local_macro_def_scopes[&id] } else { - let module_def_id = ty::DefIdTree::parent(&*self, def_id).unwrap(); - self.get_module(module_def_id) + // This is not entirely correct - a `macro_rules!` macro may occur + // inside a 'block' module: + // + // ```rust + // const _: () = { + // #[macro_export] + // macro_rules! my_macro { + // () => {}; + // } + // ` + // We don't record this information for external crates, so + // the module we compute here will be the closest 'mod' item + // (not necesssarily the actual parent of the `macro_rules!` + // macro). `macro_rules!` macros can't use def-site hygiene, + // so this hopefully won't be a problem. + // + // See https://github.com/rust-lang/rust/pull/77984#issuecomment-712445508 + self.nearest_mod_parent(def_id) } } @@ -310,10 +344,10 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { fn block_needs_anonymous_module(&mut self, block: &Block) -> bool { // If any statements are items, we need to create an anonymous module - block.stmts.iter().any(|statement| match statement.kind { - StmtKind::Item(_) | StmtKind::MacCall(_) => true, - _ => false, - }) + block + .stmts + .iter() + .any(|statement| matches!(statement.kind, StmtKind::Item(_) | StmtKind::MacCall(_))) } // Add an import to the current module. @@ -613,12 +647,21 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { /// Constructs the reduced graph for one item. fn build_reduced_graph_for_item(&mut self, item: &'b Item) { + if matches!(item.kind, ItemKind::Mod(..)) && item.ident.name == kw::Invalid { + // Fake crate root item from expand. + return; + } + let parent_scope = &self.parent_scope; let parent = parent_scope.module; let expansion = parent_scope.expansion; let ident = item.ident; let sp = item.span; let vis = self.resolve_visibility(&item.vis); + let local_def_id = self.r.local_def_id(item.id); + let def_id = local_def_id.to_def_id(); + + self.r.visibilities.insert(local_def_id, vis); match item.kind { ItemKind::Use(ref use_tree) => { @@ -651,10 +694,12 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { } else if orig_name == Some(kw::SelfLower) { self.r.graph_root } else { - let def_id = self.r.local_def_id(item.id); - let crate_id = - self.r.crate_loader.process_extern_crate(item, &self.r.definitions, def_id); - self.r.extern_crate_map.insert(def_id, crate_id); + let crate_id = self.r.crate_loader.process_extern_crate( + item, + &self.r.definitions, + local_def_id, + ); + self.r.extern_crate_map.insert(local_def_id, crate_id); self.r.get_module(DefId { krate: crate_id, index: CRATE_DEF_INDEX }) }; @@ -705,25 +750,16 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { self.r.define(parent, ident, TypeNS, imported_binding); } - ItemKind::Mod(..) if ident.name == kw::Invalid => {} // Crate root - ItemKind::Mod(..) => { - let def_id = self.r.local_def_id(item.id); - let module_kind = ModuleKind::Def(DefKind::Mod, def_id.to_def_id(), ident.name); + let module_kind = ModuleKind::Def(DefKind::Mod, def_id, ident.name); let module = self.r.arenas.alloc_module(ModuleData { no_implicit_prelude: parent.no_implicit_prelude || { self.r.session.contains_name(&item.attrs, sym::no_implicit_prelude) }, - ..ModuleData::new( - Some(parent), - module_kind, - def_id.to_def_id(), - expansion, - item.span, - ) + ..ModuleData::new(Some(parent), module_kind, def_id, expansion, item.span) }); self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion)); - self.r.module_map.insert(def_id, module); + self.r.module_map.insert(local_def_id, module); // Descend into the module. self.parent_scope.module = module; @@ -731,15 +767,15 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { // These items live in the value namespace. ItemKind::Static(..) => { - let res = Res::Def(DefKind::Static, self.r.local_def_id(item.id).to_def_id()); + let res = Res::Def(DefKind::Static, def_id); self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion)); } ItemKind::Const(..) => { - let res = Res::Def(DefKind::Const, self.r.local_def_id(item.id).to_def_id()); + let res = Res::Def(DefKind::Const, def_id); self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion)); } ItemKind::Fn(..) => { - let res = Res::Def(DefKind::Fn, self.r.local_def_id(item.id).to_def_id()); + let res = Res::Def(DefKind::Fn, def_id); self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion)); // Functions introducing procedural macros reserve a slot @@ -749,13 +785,11 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { // These items live in the type namespace. ItemKind::TyAlias(..) => { - let res = Res::Def(DefKind::TyAlias, self.r.local_def_id(item.id).to_def_id()); + let res = Res::Def(DefKind::TyAlias, def_id); self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion)); } ItemKind::Enum(_, _) => { - let def_id = self.r.local_def_id(item.id).to_def_id(); - self.r.variant_vis.insert(def_id, vis); let module_kind = ModuleKind::Def(DefKind::Enum, def_id, ident.name); let module = self.r.new_module( parent, @@ -769,14 +803,13 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { } ItemKind::TraitAlias(..) => { - let res = Res::Def(DefKind::TraitAlias, self.r.local_def_id(item.id).to_def_id()); + let res = Res::Def(DefKind::TraitAlias, def_id); self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion)); } // These items live in both the type and value namespaces. ItemKind::Struct(ref vdata, _) => { // Define a name in the type namespace. - let def_id = self.r.local_def_id(item.id).to_def_id(); let res = Res::Def(DefKind::Struct, def_id); self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion)); @@ -810,17 +843,19 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { } ret_fields.push(field_vis); } + let ctor_def_id = self.r.local_def_id(ctor_node_id); let ctor_res = Res::Def( DefKind::Ctor(CtorOf::Struct, CtorKind::from_ast(vdata)), - self.r.local_def_id(ctor_node_id).to_def_id(), + ctor_def_id.to_def_id(), ); self.r.define(parent, ident, ValueNS, (ctor_res, ctor_vis, sp, expansion)); + self.r.visibilities.insert(ctor_def_id, ctor_vis); + self.r.struct_constructors.insert(def_id, (ctor_res, ctor_vis, ret_fields)); } } ItemKind::Union(ref vdata, _) => { - let def_id = self.r.local_def_id(item.id).to_def_id(); let res = Res::Def(DefKind::Union, def_id); self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion)); @@ -829,8 +864,6 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { } ItemKind::Trait(..) => { - let def_id = self.r.local_def_id(item.id).to_def_id(); - // Add all the items within to a new module. let module_kind = ModuleKind::Def(DefKind::Trait, def_id, ident.name); let module = self.r.new_module( @@ -845,6 +878,9 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { } // These items do not add names to modules. + ItemKind::Impl { of_trait: Some(..), .. } => { + self.r.trait_impl_items.insert(local_def_id); + } ItemKind::Impl { .. } | ItemKind::ForeignMod(..) | ItemKind::GlobalAsm(..) => {} ItemKind::MacroDef(..) | ItemKind::MacCall(_) => unreachable!(), @@ -853,22 +889,20 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { /// Constructs the reduced graph for one foreign item. fn build_reduced_graph_for_foreign_item(&mut self, item: &ForeignItem) { - let (res, ns) = match item.kind { - ForeignItemKind::Fn(..) => { - (Res::Def(DefKind::Fn, self.r.local_def_id(item.id).to_def_id()), ValueNS) - } - ForeignItemKind::Static(..) => { - (Res::Def(DefKind::Static, self.r.local_def_id(item.id).to_def_id()), ValueNS) - } - ForeignItemKind::TyAlias(..) => { - (Res::Def(DefKind::ForeignTy, self.r.local_def_id(item.id).to_def_id()), TypeNS) - } + let local_def_id = self.r.local_def_id(item.id); + let def_id = local_def_id.to_def_id(); + let (def_kind, ns) = match item.kind { + ForeignItemKind::Fn(..) => (DefKind::Fn, ValueNS), + ForeignItemKind::Static(..) => (DefKind::Static, ValueNS), + ForeignItemKind::TyAlias(..) => (DefKind::ForeignTy, TypeNS), ForeignItemKind::MacCall(_) => unreachable!(), }; let parent = self.parent_scope.module; let expansion = self.parent_scope.expansion; let vis = self.resolve_visibility(&item.vis); + let res = Res::Def(def_kind, def_id); self.r.define(parent, item.ident, ns, (res, vis, item.span, expansion)); + self.r.visibilities.insert(local_def_id, vis); } fn build_reduced_graph_for_block(&mut self, block: &Block) { @@ -1205,6 +1239,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { self.r.check_reserved_macro_name(ident, res); self.insert_unused_macro(ident, def_id, item.id, span); } + self.r.visibilities.insert(def_id, vis); MacroRulesScope::Binding(self.r.arenas.alloc_macro_rules_binding(MacroRulesBinding { parent_macro_rules_scope: parent_scope.macro_rules, binding, @@ -1224,6 +1259,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> { self.insert_unused_macro(ident, def_id, item.id, span); } self.r.define(module, ident, MacroNS, (res, vis, span, expansion)); + self.r.visibilities.insert(def_id, vis); self.parent_scope.macro_rules } } @@ -1297,36 +1333,64 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> { } fn visit_assoc_item(&mut self, item: &'b AssocItem, ctxt: AssocCtxt) { - let parent = self.parent_scope.module; - if let AssocItemKind::MacCall(_) = item.kind { self.visit_invoc(item.id); return; } - if let AssocCtxt::Impl = ctxt { - self.resolve_visibility(&item.vis); - visit::walk_assoc_item(self, item, ctxt); - return; - } + let local_def_id = self.r.local_def_id(item.id); + let def_id = local_def_id.to_def_id(); + let vis = match ctxt { + AssocCtxt::Trait => { + let (def_kind, ns) = match item.kind { + AssocItemKind::Const(..) => (DefKind::AssocConst, ValueNS), + AssocItemKind::Fn(_, ref sig, _, _) => { + if sig.decl.has_self() { + self.r.has_self.insert(def_id); + } + (DefKind::AssocFn, ValueNS) + } + AssocItemKind::TyAlias(..) => (DefKind::AssocTy, TypeNS), + AssocItemKind::MacCall(_) => bug!(), // handled above + }; - // Add the item to the trait info. - let item_def_id = self.r.local_def_id(item.id).to_def_id(); - let (res, ns) = match item.kind { - AssocItemKind::Const(..) => (Res::Def(DefKind::AssocConst, item_def_id), ValueNS), - AssocItemKind::Fn(_, ref sig, _, _) => { - if sig.decl.has_self() { - self.r.has_self.insert(item_def_id); + let parent = self.parent_scope.module; + let expansion = self.parent_scope.expansion; + let res = Res::Def(def_kind, def_id); + // Trait item visibility is inherited from its trait when not specified explicitly. + let vis = match &item.vis.kind { + ast::VisibilityKind::Inherited => { + self.r.visibilities[&parent.def_id().unwrap().expect_local()] + } + _ => self.resolve_visibility(&item.vis), + }; + // FIXME: For historical reasons the binding visibility is set to public, + // use actual visibility here instead, using enum variants as an example. + let vis_hack = ty::Visibility::Public; + self.r.define(parent, item.ident, ns, (res, vis_hack, item.span, expansion)); + Some(vis) + } + AssocCtxt::Impl => { + // Trait impl item visibility is inherited from its trait when not specified + // explicitly. In that case we cannot determine it here in early resolve, + // so we leave a hole in the visibility table to be filled later. + // Inherent impl item visibility is never inherited from other items. + if matches!(item.vis.kind, ast::VisibilityKind::Inherited) + && self + .r + .trait_impl_items + .contains(&ty::DefIdTree::parent(&*self.r, def_id).unwrap().expect_local()) + { + None + } else { + Some(self.resolve_visibility(&item.vis)) } - (Res::Def(DefKind::AssocFn, item_def_id), ValueNS) } - AssocItemKind::TyAlias(..) => (Res::Def(DefKind::AssocTy, item_def_id), TypeNS), - AssocItemKind::MacCall(_) => bug!(), // handled above }; - let vis = ty::Visibility::Public; - let expansion = self.parent_scope.expansion; - self.r.define(parent, item.ident, ns, (res, vis, item.span, expansion)); + if let Some(vis) = vis { + self.r.visibilities.insert(local_def_id, vis); + } visit::walk_assoc_item(self, item, ctxt); } @@ -1394,7 +1458,8 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> { if sf.is_placeholder { self.visit_invoc(sf.id); } else { - self.resolve_visibility(&sf.vis); + let vis = self.resolve_visibility(&sf.vis); + self.r.visibilities.insert(self.r.local_def_id(sf.id), vis); visit::walk_struct_field(self, sf); } } @@ -1408,22 +1473,30 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> { } let parent = self.parent_scope.module; - let vis = self.r.variant_vis[&parent.def_id().expect("enum without def-id")]; + let vis = match variant.vis.kind { + // Variant visibility is inherited from its enum when not specified explicitly. + ast::VisibilityKind::Inherited => { + self.r.visibilities[&parent.def_id().unwrap().expect_local()] + } + _ => self.resolve_visibility(&variant.vis), + }; let expn_id = self.parent_scope.expansion; let ident = variant.ident; // Define a name in the type namespace. - let def_id = self.r.local_def_id(variant.id).to_def_id(); - let res = Res::Def(DefKind::Variant, def_id); + let def_id = self.r.local_def_id(variant.id); + let res = Res::Def(DefKind::Variant, def_id.to_def_id()); self.r.define(parent, ident, TypeNS, (res, vis, variant.span, expn_id)); + self.r.visibilities.insert(def_id, vis); - // If the variant is marked as non_exhaustive then lower the visibility to within the - // crate. - let mut ctor_vis = vis; - let has_non_exhaustive = self.r.session.contains_name(&variant.attrs, sym::non_exhaustive); - if has_non_exhaustive && vis == ty::Visibility::Public { - ctor_vis = ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)); - } + // If the variant is marked as non_exhaustive then lower the visibility to within the crate. + let ctor_vis = if vis == ty::Visibility::Public + && self.r.session.contains_name(&variant.attrs, sym::non_exhaustive) + { + ty::Visibility::Restricted(DefId::local(CRATE_DEF_INDEX)) + } else { + vis + }; // Define a constructor name in the value namespace. // Braced variants, unlike structs, generate unusable names in @@ -1431,12 +1504,15 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> { // It's ok to use the variant's id as a ctor id since an // error will be reported on any use of such resolution anyway. let ctor_node_id = variant.data.ctor_id().unwrap_or(variant.id); - let ctor_def_id = self.r.local_def_id(ctor_node_id).to_def_id(); + let ctor_def_id = self.r.local_def_id(ctor_node_id); let ctor_kind = CtorKind::from_ast(&variant.data); - let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Variant, ctor_kind), ctor_def_id); + let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Variant, ctor_kind), ctor_def_id.to_def_id()); self.r.define(parent, ident, ValueNS, (ctor_res, ctor_vis, variant.span, expn_id)); + if ctor_def_id != def_id { + self.r.visibilities.insert(ctor_def_id, ctor_vis); + } // Record field names for error reporting. - self.insert_field_names_local(ctor_def_id, &variant.data); + self.insert_field_names_local(ctor_def_id.to_def_id(), &variant.data); visit::walk_variant(self, variant); } diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs index 5d5088de31b..a4de4d500f5 100644 --- a/compiler/rustc_resolve/src/def_collector.rs +++ b/compiler/rustc_resolve/src/def_collector.rs @@ -76,6 +76,7 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> { let def_data = match &i.kind { ItemKind::Impl { .. } => DefPathData::Impl, ItemKind::Mod(..) if i.ident.name == kw::Invalid => { + // Fake crate root item from expand. return visit::walk_item(self, i); } ItemKind::Mod(..) @@ -239,13 +240,13 @@ impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> { fn visit_ty(&mut self, ty: &'a Ty) { match ty.kind { - TyKind::MacCall(..) => return self.visit_macro_invoc(ty.id), + TyKind::MacCall(..) => self.visit_macro_invoc(ty.id), TyKind::ImplTrait(node_id, _) => { - self.create_def(node_id, DefPathData::ImplTrait, ty.span); + let parent_def = self.create_def(node_id, DefPathData::ImplTrait, ty.span); + self.with_parent(parent_def, |this| visit::walk_ty(this, ty)); } - _ => {} + _ => visit::walk_ty(self, ty), } - visit::walk_ty(self, ty); } fn visit_stmt(&mut self, stmt: &'a Stmt) { diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs index 33ab09a8f42..4e115c62c9e 100644 --- a/compiler/rustc_resolve/src/diagnostics.rs +++ b/compiler/rustc_resolve/src/diagnostics.rs @@ -922,15 +922,10 @@ impl<'a> Resolver<'a> { ); self.add_typo_suggestion(err, suggestion, ident.span); - let import_suggestions = self.lookup_import_candidates( - ident, - Namespace::MacroNS, - parent_scope, - |res| match res { - Res::Def(DefKind::Macro(MacroKind::Bang), _) => true, - _ => false, - }, - ); + let import_suggestions = + self.lookup_import_candidates(ident, Namespace::MacroNS, parent_scope, |res| { + matches!(res, Res::Def(DefKind::Macro(MacroKind::Bang), _)) + }); show_candidates(err, None, &import_suggestions, false, true); if macro_kind == MacroKind::Derive && (ident.name == sym::Send || ident.name == sym::Sync) { @@ -1010,11 +1005,9 @@ impl<'a> Resolver<'a> { fn binding_description(&self, b: &NameBinding<'_>, ident: Ident, from_prelude: bool) -> String { let res = b.res(); if b.span.is_dummy() { - let add_built_in = match b.res() { - // These already contain the "built-in" prefix or look bad with it. - Res::NonMacroAttr(..) | Res::PrimTy(..) | Res::ToolMod => false, - _ => true, - }; + // These already contain the "built-in" prefix or look bad with it. + let add_built_in = + !matches!(b.res(), Res::NonMacroAttr(..) | Res::PrimTy(..) | Res::ToolMod); let (built_in, from) = if from_prelude { ("", " from prelude") } else if b.is_extern_crate() @@ -1610,10 +1603,7 @@ fn find_span_immediately_after_crate_name( if *c == ':' { num_colons += 1; } - match c { - ':' if num_colons == 2 => false, - _ => true, - } + !matches!(c, ':' if num_colons == 2) }); // Find everything after the second colon.. `foo::{baz, makro};` let from_second_colon = use_span.with_lo(until_second_colon.hi() + BytePos(1)); diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs index adff4542b0f..269d25be0a5 100644 --- a/compiler/rustc_resolve/src/imports.rs +++ b/compiler/rustc_resolve/src/imports.rs @@ -114,10 +114,7 @@ crate struct Import<'a> { impl<'a> Import<'a> { pub fn is_glob(&self) -> bool { - match self.kind { - ImportKind::Glob { .. } => true, - _ => false, - } + matches!(self.kind, ImportKind::Glob { .. }) } pub fn is_nested(&self) -> bool { @@ -898,12 +895,10 @@ impl<'a, 'b> ImportResolver<'a, 'b> { let msg = "inconsistent resolution for an import"; self.r.session.span_err(import.span, msg); } - } else { - if self.r.privacy_errors.is_empty() { - let msg = "cannot determine resolution for the import"; - let msg_note = "import resolution is stuck, try simplifying other imports"; - self.r.session.struct_span_err(import.span, msg).note(msg_note).emit(); - } + } else if self.r.privacy_errors.is_empty() { + let msg = "cannot determine resolution for the import"; + let msg_note = "import resolution is stuck, try simplifying other imports"; + self.r.session.struct_span_err(import.span, msg).note(msg_note).emit(); } module @@ -1044,19 +1039,14 @@ impl<'a, 'b> ImportResolver<'a, 'b> { if res != initial_res && this.ambiguity_errors.is_empty() { span_bug!(import.span, "inconsistent resolution for an import"); } - } else { - if res != Res::Err - && this.ambiguity_errors.is_empty() - && this.privacy_errors.is_empty() - { - let msg = "cannot determine resolution for the import"; - let msg_note = - "import resolution is stuck, try simplifying other imports"; - this.session - .struct_span_err(import.span, msg) - .note(msg_note) - .emit(); - } + } else if res != Res::Err + && this.ambiguity_errors.is_empty() + && this.privacy_errors.is_empty() + { + let msg = "cannot determine resolution for the import"; + let msg_note = + "import resolution is stuck, try simplifying other imports"; + this.session.struct_span_err(import.span, msg).note(msg_note).emit(); } } Err(..) => { diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 7517ab66170..2337f0d09ab 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -57,6 +57,12 @@ enum PatternSource { FnParam, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum IsRepeatExpr { + No, + Yes, +} + impl PatternSource { fn descr(self) -> &'static str { match self { @@ -251,16 +257,12 @@ impl<'a> PathSource<'a> { } fn is_call(self) -> bool { - match self { - PathSource::Expr(Some(&Expr { kind: ExprKind::Call(..), .. })) => true, - _ => false, - } + matches!(self, PathSource::Expr(Some(&Expr { kind: ExprKind::Call(..), .. }))) } crate fn is_expected(self, res: Res) -> bool { match self { - PathSource::Type => match res { - Res::Def( + PathSource::Type => matches!(res, Res::Def( DefKind::Struct | DefKind::Union | DefKind::Enum @@ -274,19 +276,12 @@ impl<'a> PathSource<'a> { _, ) | Res::PrimTy(..) - | Res::SelfTy(..) => true, - _ => false, - }, - PathSource::Trait(AliasPossibility::No) => match res { - Res::Def(DefKind::Trait, _) => true, - _ => false, - }, - PathSource::Trait(AliasPossibility::Maybe) => match res { - Res::Def(DefKind::Trait | DefKind::TraitAlias, _) => true, - _ => false, - }, - PathSource::Expr(..) => match res { - Res::Def( + | Res::SelfTy(..)), + PathSource::Trait(AliasPossibility::No) => matches!(res, Res::Def(DefKind::Trait, _)), + PathSource::Trait(AliasPossibility::Maybe) => { + matches!(res, Res::Def(DefKind::Trait | DefKind::TraitAlias, _)) + } + PathSource::Expr(..) => matches!(res, Res::Def( DefKind::Ctor(_, CtorKind::Const | CtorKind::Fn) | DefKind::Const | DefKind::Static @@ -297,23 +292,16 @@ impl<'a> PathSource<'a> { _, ) | Res::Local(..) - | Res::SelfCtor(..) => true, - _ => false, - }, - PathSource::Pat => match res { - Res::Def( + | Res::SelfCtor(..)), + PathSource::Pat => matches!(res, Res::Def( DefKind::Ctor(_, CtorKind::Const) | DefKind::Const | DefKind::AssocConst, _, ) - | Res::SelfCtor(..) => true, - _ => false, - }, - PathSource::TupleStruct(..) => match res { - Res::Def(DefKind::Ctor(_, CtorKind::Fn), _) | Res::SelfCtor(..) => true, - _ => false, - }, - PathSource::Struct => match res { - Res::Def( + | Res::SelfCtor(..)), + PathSource::TupleStruct(..) => { + matches!(res, Res::Def(DefKind::Ctor(_, CtorKind::Fn), _) | Res::SelfCtor(..)) + } + PathSource::Struct => matches!(res, Res::Def( DefKind::Struct | DefKind::Union | DefKind::Variant @@ -321,9 +309,7 @@ impl<'a> PathSource<'a> { | DefKind::AssocTy, _, ) - | Res::SelfTy(..) => true, - _ => false, - }, + | Res::SelfTy(..)), PathSource::TraitItem(ns) => match res { Res::Def(DefKind::AssocConst | DefKind::AssocFn, _) if ns == ValueNS => true, Res::Def(DefKind::AssocTy, _) if ns == TypeNS => true, @@ -353,8 +339,8 @@ impl<'a> PathSource<'a> { #[derive(Default)] struct DiagnosticMetadata<'ast> { - /// The current trait's associated types' ident, used for diagnostic suggestions. - current_trait_assoc_types: Vec<Ident>, + /// The current trait's associated items' ident, used for diagnostic suggestions. + current_trait_assoc_items: Option<&'ast [P<AssocItem>]>, /// The current self type if inside an impl (used for better errors). current_self_type: Option<Ty>, @@ -437,10 +423,8 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { self.resolve_block(block); } fn visit_anon_const(&mut self, constant: &'ast AnonConst) { - debug!("visit_anon_const {:?}", constant); - self.with_constant_rib(constant.value.is_potential_trivial_const_param(), |this| { - visit::walk_anon_const(this, constant); - }); + // We deal with repeat expressions explicitly in `resolve_expr`. + self.resolve_anon_const(constant, IsRepeatExpr::No); } fn visit_expr(&mut self, expr: &'ast Expr) { self.resolve_expr(expr, None); @@ -647,7 +631,11 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> { if !check_ns(TypeNS) && check_ns(ValueNS) { // This must be equivalent to `visit_anon_const`, but we cannot call it // directly due to visitor lifetimes so we have to copy-paste some code. - self.with_constant_rib(true, |this| { + // + // Note that we might not be inside of an repeat expression here, + // but considering that `IsRepeatExpr` is only relevant for + // non-trivial constants this is doesn't matter. + self.with_constant_rib(IsRepeatExpr::No, true, |this| { this.smart_resolve_path( ty.id, qself.as_ref(), @@ -980,9 +968,11 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // // Type parameters can already be used and as associated consts are // not used as part of the type system, this is far less surprising. - this.with_constant_rib(true, |this| { - this.visit_expr(expr) - }); + this.with_constant_rib( + IsRepeatExpr::No, + true, + |this| this.visit_expr(expr), + ); } } AssocItemKind::Fn(_, _, generics, _) => { @@ -1023,7 +1013,9 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { self.with_item_rib(HasGenericParams::No, |this| { this.visit_ty(ty); if let Some(expr) = expr { - this.with_constant_rib(expr.is_potential_trivial_const_param(), |this| { + // We already forbid generic params because of the above item rib, + // so it doesn't matter whether this is a trivial constant. + this.with_constant_rib(IsRepeatExpr::No, true, |this| { this.visit_expr(expr) }); } @@ -1122,12 +1114,29 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { self.with_rib(ValueNS, kind, |this| this.with_rib(TypeNS, kind, f)) } - fn with_constant_rib(&mut self, trivial: bool, f: impl FnOnce(&mut Self)) { - debug!("with_constant_rib"); - self.with_rib(ValueNS, ConstantItemRibKind(trivial), |this| { - this.with_rib(TypeNS, ConstantItemRibKind(trivial), |this| { - this.with_label_rib(ConstantItemRibKind(trivial), f); - }) + // HACK(min_const_generics,const_evaluatable_unchecked): We + // want to keep allowing `[0; std::mem::size_of::<*mut T>()]` + // with a future compat lint for now. We do this by adding an + // additional special case for repeat expressions. + // + // Note that we intentionally still forbid `[0; N + 1]` during + // name resolution so that we don't extend the future + // compat lint to new cases. + fn with_constant_rib( + &mut self, + is_repeat: IsRepeatExpr, + is_trivial: bool, + f: impl FnOnce(&mut Self), + ) { + debug!("with_constant_rib: is_repeat={:?} is_trivial={}", is_repeat, is_trivial); + self.with_rib(ValueNS, ConstantItemRibKind(is_trivial), |this| { + this.with_rib( + TypeNS, + ConstantItemRibKind(is_repeat == IsRepeatExpr::Yes || is_trivial), + |this| { + this.with_label_rib(ConstantItemRibKind(is_trivial), f); + }, + ) }); } @@ -1148,26 +1157,18 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { result } - /// When evaluating a `trait` use its associated types' idents for suggestionsa in E0412. + /// When evaluating a `trait` use its associated types' idents for suggestions in E0412. fn with_trait_items<T>( &mut self, - trait_items: &Vec<P<AssocItem>>, + trait_items: &'ast Vec<P<AssocItem>>, f: impl FnOnce(&mut Self) -> T, ) -> T { - let trait_assoc_types = replace( - &mut self.diagnostic_metadata.current_trait_assoc_types, - trait_items - .iter() - .filter_map(|item| match &item.kind { - AssocItemKind::TyAlias(_, _, bounds, _) if bounds.is_empty() => { - Some(item.ident) - } - _ => None, - }) - .collect(), + let trait_assoc_items = replace( + &mut self.diagnostic_metadata.current_trait_assoc_items, + Some(&trait_items[..]), ); let result = f(self); - self.diagnostic_metadata.current_trait_assoc_types = trait_assoc_types; + self.diagnostic_metadata.current_trait_assoc_items = trait_assoc_items; result } @@ -1272,9 +1273,17 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { // // Type parameters can already be used and as associated consts are // not used as part of the type system, this is far less surprising. - this.with_constant_rib(true, |this| { - visit::walk_assoc_item(this, item, AssocCtxt::Impl) - }); + this.with_constant_rib( + IsRepeatExpr::No, + true, + |this| { + visit::walk_assoc_item( + this, + item, + AssocCtxt::Impl, + ) + }, + ); } AssocItemKind::Fn(_, _, generics, _) => { // We also need a new scope for the impl item type parameters. @@ -1413,10 +1422,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { } fn is_base_res_local(&self, nid: NodeId) -> bool { - match self.r.partial_res_map.get(&nid).map(|res| res.base_res()) { - Some(Res::Local(..)) => true, - _ => false, - } + matches!(self.r.partial_res_map.get(&nid).map(|res| res.base_res()), Some(Res::Local(..))) } /// Checks that all of the arms in an or-pattern have exactly the @@ -2199,6 +2205,17 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { debug!("(resolving block) leaving block"); } + fn resolve_anon_const(&mut self, constant: &'ast AnonConst, is_repeat: IsRepeatExpr) { + debug!("resolve_anon_const {:?} is_repeat: {:?}", constant, is_repeat); + self.with_constant_rib( + is_repeat, + constant.value.is_potential_trivial_const_param(), + |this| { + visit::walk_anon_const(this, constant); + }, + ); + } + fn resolve_expr(&mut self, expr: &'ast Expr, parent: Option<&'ast Expr>) { // First, record candidate traits for this expression if it could // result in the invocation of a method call. @@ -2322,6 +2339,10 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { ExprKind::Async(..) | ExprKind::Closure(..) => { self.with_label_rib(ClosureOrAsyncRibKind, |this| visit::walk_expr(this, expr)); } + ExprKind::Repeat(ref elem, ref ct) => { + self.visit_expr(elem); + self.resolve_anon_const(ct, IsRepeatExpr::Yes); + } _ => { visit::walk_expr(self, expr); } diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index c24b383f3b8..a3e0028dc75 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -30,7 +30,21 @@ type Res = def::Res<ast::NodeId>; enum AssocSuggestion { Field, MethodWithSelf, - AssocItem, + AssocFn, + AssocType, + AssocConst, +} + +impl AssocSuggestion { + fn action(&self) -> &'static str { + match self { + AssocSuggestion::Field => "use the available field", + AssocSuggestion::MethodWithSelf => "call the method with the fully-qualified path", + AssocSuggestion::AssocFn => "call the associated function", + AssocSuggestion::AssocConst => "use the associated `const`", + AssocSuggestion::AssocType => "use the associated type", + } + } } crate enum MissingLifetimeSpot<'tcx> { @@ -386,15 +400,18 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { AssocSuggestion::MethodWithSelf if self_is_available => { err.span_suggestion( span, - "try", + "you might have meant to call the method", format!("self.{}", path_str), Applicability::MachineApplicable, ); } - AssocSuggestion::MethodWithSelf | AssocSuggestion::AssocItem => { + AssocSuggestion::MethodWithSelf + | AssocSuggestion::AssocFn + | AssocSuggestion::AssocConst + | AssocSuggestion::AssocType => { err.span_suggestion( span, - "try", + &format!("you might have meant to {}", candidate.action()), format!("Self::{}", path_str), Applicability::MachineApplicable, ); @@ -702,10 +719,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { _ => break, } } - let followed_by_brace = match sm.span_to_snippet(sp) { - Ok(ref snippet) if snippet == "{" => true, - _ => false, - }; + let followed_by_brace = matches!(sm.span_to_snippet(sp), Ok(ref snippet) if snippet == "{"); // In case this could be a struct literal that needs to be surrounded // by parentheses, find the appropriate span. let mut i = 0; @@ -917,54 +931,71 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { self.suggest_using_enum_variant(err, source, def_id, span); } (Res::Def(DefKind::Struct, def_id), _) if ns == ValueNS => { - if let Some((ctor_def, ctor_vis, fields)) = - self.r.struct_constructors.get(&def_id).cloned() + let (ctor_def, ctor_vis, fields) = + if let Some(struct_ctor) = self.r.struct_constructors.get(&def_id).cloned() { + struct_ctor + } else { + bad_struct_syntax_suggestion(def_id); + return true; + }; + + let is_accessible = self.r.is_accessible_from(ctor_vis, self.parent_scope.module); + if !is_expected(ctor_def) || is_accessible { + return true; + } + + let field_spans = match source { + // e.g. `if let Enum::TupleVariant(field1, field2) = _` + PathSource::TupleStruct(_, pattern_spans) => { + err.set_primary_message( + "cannot match against a tuple struct which contains private fields", + ); + + // Use spans of the tuple struct pattern. + Some(Vec::from(pattern_spans)) + } + // e.g. `let _ = Enum::TupleVariant(field1, field2);` + _ if source.is_call() => { + err.set_primary_message( + "cannot initialize a tuple struct which contains private fields", + ); + + // Use spans of the tuple struct definition. + self.r + .field_names + .get(&def_id) + .map(|fields| fields.iter().map(|f| f.span).collect::<Vec<_>>()) + } + _ => None, + }; + + if let Some(spans) = + field_spans.filter(|spans| spans.len() > 0 && fields.len() == spans.len()) { - let accessible_ctor = - self.r.is_accessible_from(ctor_vis, self.parent_scope.module); - if is_expected(ctor_def) && !accessible_ctor { - let mut better_diag = false; - if let PathSource::TupleStruct(_, pattern_spans) = source { - if pattern_spans.len() > 0 && fields.len() == pattern_spans.len() { - let non_visible_spans: Vec<Span> = fields - .iter() - .zip(pattern_spans.iter()) - .filter_map(|(vis, span)| { - match self - .r - .is_accessible_from(*vis, self.parent_scope.module) - { - true => None, - false => Some(*span), - } - }) - .collect(); - // Extra check to be sure - if non_visible_spans.len() > 0 { - let mut m: rustc_span::MultiSpan = - non_visible_spans.clone().into(); - non_visible_spans.into_iter().for_each(|s| { - m.push_span_label(s, "private field".to_string()) - }); - err.span_note( - m, - "constructor is not visible here due to private fields", - ); - better_diag = true; - } - } - } + let non_visible_spans: Vec<Span> = fields + .iter() + .zip(spans.iter()) + .filter(|(vis, _)| { + !self.r.is_accessible_from(**vis, self.parent_scope.module) + }) + .map(|(_, span)| *span) + .collect(); - if !better_diag { - err.span_label( - span, - "constructor is not visible here due to private fields".to_string(), - ); - } + if non_visible_spans.len() > 0 { + let mut m: rustc_span::MultiSpan = non_visible_spans.clone().into(); + non_visible_spans + .into_iter() + .for_each(|s| m.push_span_label(s, "private field".to_string())); + err.span_note(m, "constructor is not visible here due to private fields"); } - } else { - bad_struct_syntax_suggestion(def_id); + + return true; } + + err.span_label( + span, + "constructor is not visible here due to private fields".to_string(), + ); } ( Res::Def( @@ -1048,9 +1079,19 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { } } - for assoc_type_ident in &self.diagnostic_metadata.current_trait_assoc_types { - if *assoc_type_ident == ident { - return Some(AssocSuggestion::AssocItem); + if let Some(items) = self.diagnostic_metadata.current_trait_assoc_items { + for assoc_item in &items[..] { + if assoc_item.ident == ident { + return Some(match &assoc_item.kind { + ast::AssocItemKind::Const(..) => AssocSuggestion::AssocConst, + ast::AssocItemKind::Fn(_, sig, ..) if sig.decl.has_self() => { + AssocSuggestion::MethodWithSelf + } + ast::AssocItemKind::Fn(..) => AssocSuggestion::AssocFn, + ast::AssocItemKind::TyAlias(..) => AssocSuggestion::AssocType, + ast::AssocItemKind::MacCall(_) => continue, + }); + } } } @@ -1066,11 +1107,20 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> { ) { let res = binding.res(); if filter_fn(res) { - return Some(if self.r.has_self.contains(&res.def_id()) { - AssocSuggestion::MethodWithSelf + if self.r.has_self.contains(&res.def_id()) { + return Some(AssocSuggestion::MethodWithSelf); } else { - AssocSuggestion::AssocItem - }); + match res { + Res::Def(DefKind::AssocFn, _) => return Some(AssocSuggestion::AssocFn), + Res::Def(DefKind::AssocConst, _) => { + return Some(AssocSuggestion::AssocConst); + } + Res::Def(DefKind::AssocTy, _) => { + return Some(AssocSuggestion::AssocType); + } + _ => {} + } + } } } } @@ -1771,12 +1821,11 @@ impl<'tcx> LifetimeContext<'_, 'tcx> { } msg = "consider introducing a named lifetime parameter".to_string(); should_break = true; - if let Some(param) = generics.params.iter().find(|p| match p.kind { - hir::GenericParamKind::Type { + if let Some(param) = generics.params.iter().find(|p| { + !matches!(p.kind, hir::GenericParamKind::Type { synthetic: Some(hir::SyntheticTyParamKind::ImplTrait), .. - } => false, - _ => true, + }) }) { (param.span.shrink_to_lo(), "'a, ".to_string()) } else { diff --git a/compiler/rustc_resolve/src/late/lifetimes.rs b/compiler/rustc_resolve/src/late/lifetimes.rs index 072fb509b19..c79d670737e 100644 --- a/compiler/rustc_resolve/src/late/lifetimes.rs +++ b/compiler/rustc_resolve/src/late/lifetimes.rs @@ -351,10 +351,7 @@ fn krate(tcx: TyCtxt<'_>) -> NamedRegionMap { /// We have to account for this when computing the index of the other generic parameters. /// This function returns whether there is such an implicit parameter defined on the given item. fn sub_items_have_self_param(node: &hir::ItemKind<'_>) -> bool { - match *node { - hir::ItemKind::Trait(..) | hir::ItemKind::TraitAlias(..) => true, - _ => false, - } + matches!(*node, hir::ItemKind::Trait(..) | hir::ItemKind::TraitAlias(..)) } impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { @@ -417,10 +414,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { // Impls permit `'_` to be used and it is equivalent to "some fresh lifetime name". // This is not true for other kinds of items.x - let track_lifetime_uses = match item.kind { - hir::ItemKind::Impl { .. } => true, - _ => false, - }; + let track_lifetime_uses = matches!(item.kind, hir::ItemKind::Impl { .. }); // These kinds of items have only early-bound lifetime parameters. let mut index = if sub_items_have_self_param(&item.kind) { 1 // Self comes before lifetimes @@ -970,10 +964,10 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> { let trait_ref_hack = take(&mut self.trait_ref_hack); if !trait_ref_hack - || trait_ref.bound_generic_params.iter().any(|param| match param.kind { - GenericParamKind::Lifetime { .. } => true, - _ => false, - }) + || trait_ref + .bound_generic_params + .iter() + .any(|param| matches!(param.kind, GenericParamKind::Lifetime { .. })) { if trait_ref_hack { struct_span_err!( @@ -1384,18 +1378,16 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { } if in_band { Some(param.span) + } else if generics.params.len() == 1 { + // if sole lifetime, remove the entire `<>` brackets + Some(generics.span) } else { - if generics.params.len() == 1 { - // if sole lifetime, remove the entire `<>` brackets - Some(generics.span) + // if removing within `<>` brackets, we also want to + // delete a leading or trailing comma as appropriate + if i >= generics.params.len() - 1 { + Some(generics.params[i - 1].span.shrink_to_hi().to(param.span)) } else { - // if removing within `<>` brackets, we also want to - // delete a leading or trailing comma as appropriate - if i >= generics.params.len() - 1 { - Some(generics.params[i - 1].span.shrink_to_hi().to(param.span)) - } else { - Some(param.span.to(generics.params[i + 1].span.shrink_to_lo())) - } + Some(param.span.to(generics.params[i + 1].span.shrink_to_lo())) } } } else { @@ -2047,10 +2039,8 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> { // // This is intended to leave room for us to implement the // correct behavior in the future. - let has_lifetime_parameter = generic_args.args.iter().any(|arg| match arg { - GenericArg::Lifetime(_) => true, - _ => false, - }); + let has_lifetime_parameter = + generic_args.args.iter().any(|arg| matches!(arg, GenericArg::Lifetime(_))); // Resolve lifetimes found in the type `XX` from `Item = XX` bindings. for b in generic_args.bindings { diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs index e7486db4deb..30cd9944b1a 100644 --- a/compiler/rustc_resolve/src/lib.rs +++ b/compiler/rustc_resolve/src/lib.rs @@ -19,7 +19,7 @@ pub use rustc_hir::def::{Namespace, PerNS}; use Determinacy::*; -use rustc_arena::TypedArena; +use rustc_arena::{DroplessArena, TypedArena}; use rustc_ast::node_id::NodeMap; use rustc_ast::unwrap_or; use rustc_ast::visit::{self, Visitor}; @@ -313,17 +313,17 @@ impl<'tcx> Visitor<'tcx> for UsePlacementFinder { ItemKind::ExternCrate(_) => {} // but place them before the first other item _ => { - if self.span.map_or(true, |span| item.span < span) { - if !item.span.from_expansion() { - // don't insert between attributes and an item - if item.attrs.is_empty() { - self.span = Some(item.span.shrink_to_lo()); - } else { - // find the first attribute on the item - for attr in &item.attrs { - if self.span.map_or(true, |span| attr.span < span) { - self.span = Some(attr.span.shrink_to_lo()); - } + if self.span.map_or(true, |span| item.span < span) + && !item.span.from_expansion() + { + // don't insert between attributes and an item + if item.attrs.is_empty() { + self.span = Some(item.span.shrink_to_lo()); + } else { + // find the first attribute on the item + for attr in &item.attrs { + if self.span.map_or(true, |span| attr.span < span) { + self.span = Some(attr.span.shrink_to_lo()); } } } @@ -558,17 +558,11 @@ impl<'a> ModuleData<'a> { // `self` resolves to the first module ancestor that `is_normal`. fn is_normal(&self) -> bool { - match self.kind { - ModuleKind::Def(DefKind::Mod, _, _) => true, - _ => false, - } + matches!(self.kind, ModuleKind::Def(DefKind::Mod, _, _)) } fn is_trait(&self) -> bool { - match self.kind { - ModuleKind::Def(DefKind::Trait, _, _) => true, - _ => false, - } + matches!(self.kind, ModuleKind::Def(DefKind::Trait, _, _)) } fn nearest_item_scope(&'a self) -> Module<'a> { @@ -628,10 +622,7 @@ enum NameBindingKind<'a> { impl<'a> NameBindingKind<'a> { /// Is this a name binding of a import? fn is_import(&self) -> bool { - match *self { - NameBindingKind::Import { .. } => true, - _ => false, - } + matches!(*self, NameBindingKind::Import { .. }) } } @@ -750,13 +741,10 @@ impl<'a> NameBinding<'a> { } fn is_variant(&self) -> bool { - match self.kind { - NameBindingKind::Res( + matches!(self.kind, NameBindingKind::Res( Res::Def(DefKind::Variant | DefKind::Ctor(CtorOf::Variant, ..), _), _, - ) => true, - _ => false, - } + )) } fn is_extern_crate(&self) -> bool { @@ -774,10 +762,7 @@ impl<'a> NameBinding<'a> { } fn is_import(&self) -> bool { - match self.kind { - NameBindingKind::Import { .. } => true, - _ => false, - } + matches!(self.kind, NameBindingKind::Import { .. }) } fn is_glob_import(&self) -> bool { @@ -788,17 +773,14 @@ impl<'a> NameBinding<'a> { } fn is_importable(&self) -> bool { - match self.res() { - Res::Def(DefKind::AssocConst | DefKind::AssocFn | DefKind::AssocTy, _) => false, - _ => true, - } + !matches!( + self.res(), + Res::Def(DefKind::AssocConst | DefKind::AssocFn | DefKind::AssocTy, _) + ) } fn is_macro_def(&self) -> bool { - match self.kind { - NameBindingKind::Res(Res::Def(DefKind::Macro(..), _), _) => true, - _ => false, - } + matches!(self.kind, NameBindingKind::Res(Res::Def(DefKind::Macro(..), _), _)) } fn macro_kind(&self) -> Option<MacroKind> { @@ -944,7 +926,8 @@ pub struct Resolver<'a> { /// Maps glob imports to the names of items actually imported. glob_map: FxHashMap<LocalDefId, FxHashSet<Symbol>>, - + /// Visibilities in "lowered" form, for all entities that have them. + visibilities: FxHashMap<LocalDefId, ty::Visibility>, used_imports: FxHashSet<(NodeId, Namespace)>, maybe_unused_trait_imports: FxHashSet<LocalDefId>, maybe_unused_extern_crates: Vec<(LocalDefId, Span)>, @@ -1008,10 +991,6 @@ pub struct Resolver<'a> { /// Features enabled for this crate. active_features: FxHashSet<Symbol>, - /// Stores enum visibilities to properly build a reduced graph - /// when visiting the correspondent variants. - variant_vis: DefIdMap<ty::Visibility>, - lint_buffer: LintBuffer, next_node_id: NodeId, @@ -1028,6 +1007,9 @@ pub struct Resolver<'a> { invocation_parents: FxHashMap<ExpnId, LocalDefId>, next_disambiguator: FxHashMap<(LocalDefId, DefPathData), u32>, + /// Some way to know that we are in a *trait* impl in `visit_assoc_item`. + /// FIXME: Replace with a more general AST map (together with some other fields). + trait_impl_items: FxHashSet<LocalDefId>, } /// Nothing really interesting here; it just provides memory for the rest of the crate. @@ -1035,12 +1017,10 @@ pub struct Resolver<'a> { pub struct ResolverArenas<'a> { modules: TypedArena<ModuleData<'a>>, local_modules: RefCell<Vec<Module<'a>>>, - name_bindings: TypedArena<NameBinding<'a>>, imports: TypedArena<Import<'a>>, name_resolutions: TypedArena<RefCell<NameResolution<'a>>>, - macro_rules_bindings: TypedArena<MacroRulesBinding<'a>>, ast_paths: TypedArena<ast::Path>, - pattern_spans: TypedArena<Span>, + dropless: DroplessArena, } impl<'a> ResolverArenas<'a> { @@ -1055,7 +1035,7 @@ impl<'a> ResolverArenas<'a> { self.local_modules.borrow() } fn alloc_name_binding(&'a self, name_binding: NameBinding<'a>) -> &'a NameBinding<'a> { - self.name_bindings.alloc(name_binding) + self.dropless.alloc(name_binding) } fn alloc_import(&'a self, import: Import<'a>) -> &'a Import<'_> { self.imports.alloc(import) @@ -1067,13 +1047,13 @@ impl<'a> ResolverArenas<'a> { &'a self, binding: MacroRulesBinding<'a>, ) -> &'a MacroRulesBinding<'a> { - self.macro_rules_bindings.alloc(binding) + self.dropless.alloc(binding) } fn alloc_ast_paths(&'a self, paths: &[ast::Path]) -> &'a [ast::Path] { self.ast_paths.alloc_from_iter(paths.iter().cloned()) } fn alloc_pattern_spans(&'a self, spans: impl Iterator<Item = Span>) -> &'a [Span] { - self.pattern_spans.alloc_from_iter(spans) + self.dropless.alloc_from_iter(spans) } } @@ -1195,7 +1175,8 @@ impl<'a> Resolver<'a> { metadata_loader: &'a MetadataLoaderDyn, arenas: &'a ResolverArenas<'a>, ) -> Resolver<'a> { - let root_def_id = DefId::local(CRATE_DEF_INDEX); + let root_local_def_id = LocalDefId { local_def_index: CRATE_DEF_INDEX }; + let root_def_id = root_local_def_id.to_def_id(); let root_module_kind = ModuleKind::Def(DefKind::Mod, root_def_id, kw::Invalid); let graph_root = arenas.alloc_module(ModuleData { no_implicit_prelude: session.contains_name(&krate.attrs, sym::no_implicit_prelude), @@ -1213,11 +1194,14 @@ impl<'a> Resolver<'a> { ) }); let mut module_map = FxHashMap::default(); - module_map.insert(LocalDefId { local_def_index: CRATE_DEF_INDEX }, graph_root); + module_map.insert(root_local_def_id, graph_root); let definitions = Definitions::new(crate_name, session.local_crate_disambiguator()); let root = definitions.get_root_def(); + let mut visibilities = FxHashMap::default(); + visibilities.insert(root_local_def_id, ty::Visibility::Public); + let mut def_id_to_span = IndexVec::default(); assert_eq!(def_id_to_span.push(rustc_span::DUMMY_SP), root); let mut def_id_to_node_id = IndexVec::default(); @@ -1290,7 +1274,7 @@ impl<'a> Resolver<'a> { ast_transform_scopes: FxHashMap::default(), glob_map: Default::default(), - + visibilities, used_imports: FxHashSet::default(), maybe_unused_trait_imports: Default::default(), maybe_unused_extern_crates: Vec::new(), @@ -1339,7 +1323,6 @@ impl<'a> Resolver<'a> { .map(|(feat, ..)| *feat) .chain(features.declared_lang_features.iter().map(|(feat, ..)| *feat)) .collect(), - variant_vis: Default::default(), lint_buffer: LintBuffer::default(), next_node_id: NodeId::from_u32(1), def_id_to_span, @@ -1348,6 +1331,7 @@ impl<'a> Resolver<'a> { placeholder_field_indices: Default::default(), invocation_parents, next_disambiguator: Default::default(), + trait_impl_items: Default::default(), } } @@ -1371,14 +1355,16 @@ impl<'a> Resolver<'a> { pub fn into_outputs(self) -> ResolverOutputs { let definitions = self.definitions; + let visibilities = self.visibilities; let extern_crate_map = self.extern_crate_map; let export_map = self.export_map; let maybe_unused_trait_imports = self.maybe_unused_trait_imports; let maybe_unused_extern_crates = self.maybe_unused_extern_crates; let glob_map = self.glob_map; ResolverOutputs { - definitions: definitions, + definitions, cstore: Box::new(self.crate_loader.into_cstore()), + visibilities, extern_crate_map, export_map, glob_map, @@ -1396,6 +1382,7 @@ impl<'a> Resolver<'a> { ResolverOutputs { definitions: self.definitions.clone(), cstore: Box::new(self.cstore().clone()), + visibilities: self.visibilities.clone(), extern_crate_map: self.extern_crate_map.clone(), export_map: self.export_map.clone(), glob_map: self.glob_map.clone(), @@ -1720,10 +1707,9 @@ impl<'a> Resolver<'a> { Scope::MacroRules(binding.parent_macro_rules_scope) } MacroRulesScope::Invocation(invoc_id) => Scope::MacroRules( - self.output_macro_rules_scopes - .get(&invoc_id) - .cloned() - .unwrap_or(self.invocation_parent_scopes[&invoc_id].macro_rules), + self.output_macro_rules_scopes.get(&invoc_id).cloned().unwrap_or_else( + || self.invocation_parent_scopes[&invoc_id].macro_rules, + ), ), MacroRulesScope::Empty => Scope::Module(module), }, @@ -1988,11 +1974,12 @@ impl<'a> Resolver<'a> { // The macro is a proc macro derive if let Some(def_id) = module.expansion.expn_data().macro_def_id { if let Some(ext) = self.get_macro_by_def_id(def_id) { - if !ext.is_builtin && ext.macro_kind() == MacroKind::Derive { - if parent.expansion.outer_expn_is_descendant_of(span.ctxt()) { - *poisoned = Some(node_id); - return module.parent; - } + if !ext.is_builtin + && ext.macro_kind() == MacroKind::Derive + && parent.expansion.outer_expn_is_descendant_of(span.ctxt()) + { + *poisoned = Some(node_id); + return module.parent; } } } @@ -2386,10 +2373,7 @@ impl<'a> Resolver<'a> { _ => None, }; let (label, suggestion) = if module_res == self.graph_root.res() { - let is_mod = |res| match res { - Res::Def(DefKind::Mod, _) => true, - _ => false, - }; + let is_mod = |res| matches!(res, Res::Def(DefKind::Mod, _)); // Don't look up import candidates if this is a speculative resolve let mut candidates = if record_used { self.lookup_import_candidates(ident, TypeNS, parent_scope, is_mod) diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs index bea71389647..b5b281b93bc 100644 --- a/compiler/rustc_resolve/src/macros.rs +++ b/compiler/rustc_resolve/src/macros.rs @@ -19,7 +19,7 @@ use rustc_feature::is_builtin_attr_name; use rustc_hir::def::{self, DefKind, NonMacroAttrKind}; use rustc_hir::def_id; use rustc_middle::middle::stability; -use rustc_middle::{span_bug, ty}; +use rustc_middle::ty; use rustc_session::lint::builtin::UNUSED_MACROS; use rustc_session::Session; use rustc_span::edition::Edition; @@ -885,11 +885,11 @@ impl<'a> Resolver<'a> { initial_res: Option<Res>, res: Res| { if let Some(initial_res) = initial_res { - if res != initial_res && res != Res::Err && this.ambiguity_errors.is_empty() { + if res != initial_res { // Make sure compilation does not succeed if preferred macro resolution // has changed after the macro had been expanded. In theory all such - // situations should be reported as ambiguity errors, so this is a bug. - span_bug!(span, "inconsistent resolution for a macro"); + // situations should be reported as errors, so this is a bug. + this.session.delay_span_bug(span, "inconsistent resolution for a macro"); } } else { // It's possible that the macro was unresolved (indeterminate) and silently diff --git a/compiler/rustc_save_analysis/src/dump_visitor.rs b/compiler/rustc_save_analysis/src/dump_visitor.rs index ce484858cbb..dbb5e3cc9f0 100644 --- a/compiler/rustc_save_analysis/src/dump_visitor.rs +++ b/compiler/rustc_save_analysis/src/dump_visitor.rs @@ -320,6 +320,15 @@ impl<'tcx> DumpVisitor<'tcx> { for param in generics.params { match param.kind { hir::GenericParamKind::Lifetime { .. } => {} + hir::GenericParamKind::Type { + synthetic: Some(hir::SyntheticTyParamKind::ImplTrait), + .. + } => { + return self + .nest_typeck_results(self.tcx.hir().local_def_id(param.hir_id), |this| { + this.visit_generics(generics) + }); + } hir::GenericParamKind::Type { .. } => { let param_ss = param.name.ident().span; let name = escape(self.span.snippet(param_ss)); @@ -351,7 +360,8 @@ impl<'tcx> DumpVisitor<'tcx> { hir::GenericParamKind::Const { .. } => {} } } - self.visit_generics(generics); + + self.visit_generics(generics) } fn process_fn( diff --git a/compiler/rustc_save_analysis/src/lib.rs b/compiler/rustc_save_analysis/src/lib.rs index f6434689fec..48d15370ee3 100644 --- a/compiler/rustc_save_analysis/src/lib.rs +++ b/compiler/rustc_save_analysis/src/lib.rs @@ -630,9 +630,14 @@ impl<'tcx> SaveContext<'tcx> { }) | Node::Ty(&hir::Ty { kind: hir::TyKind::Path(ref qpath), .. }) => match qpath { hir::QPath::Resolved(_, path) => path.res, - hir::QPath::TypeRelative(..) | hir::QPath::LangItem(..) => self - .maybe_typeck_results - .map_or(Res::Err, |typeck_results| typeck_results.qpath_res(qpath, hir_id)), + hir::QPath::TypeRelative(..) | hir::QPath::LangItem(..) => { + // #75962: `self.typeck_results` may be different from the `hir_id`'s result. + if self.tcx.has_typeck_results(hir_id.owner.to_def_id()) { + self.tcx.typeck(hir_id.owner).qpath_res(qpath, hir_id) + } else { + Res::Err + } + } }, Node::Binding(&hir::Pat { diff --git a/compiler/rustc_save_analysis/src/sig.rs b/compiler/rustc_save_analysis/src/sig.rs index 747e198cd93..1bf8160e4c3 100644 --- a/compiler/rustc_save_analysis/src/sig.rs +++ b/compiler/rustc_save_analysis/src/sig.rs @@ -262,7 +262,7 @@ impl<'hir> Sig for hir::Ty<'hir> { } else { let start = offset + prefix.len() + 5; let end = start + name.len(); - // FIXME should put the proper path in there, not elipses. + // FIXME should put the proper path in there, not ellipsis. Ok(Signature { text: prefix + "...::" + &name, defs: vec![], @@ -272,7 +272,7 @@ impl<'hir> Sig for hir::Ty<'hir> { } hir::TyKind::Path(hir::QPath::TypeRelative(ty, segment)) => { let nested_ty = ty.make(offset + 1, id, scx)?; - let prefix = format!("<{}>::", nested_ty.text,); + let prefix = format!("<{}>::", nested_ty.text); let name = path_segment_to_string(segment); let res = scx.get_path_res(id.ok_or("Missing id for Path")?); @@ -551,7 +551,7 @@ impl<'hir> Sig for hir::Item<'hir> { // FIXME where clause } hir::ItemKind::ForeignMod(_) => Err("extern mod"), - hir::ItemKind::GlobalAsm(_) => Err("glboal asm"), + hir::ItemKind::GlobalAsm(_) => Err("global asm"), hir::ItemKind::ExternCrate(_) => Err("extern crate"), hir::ItemKind::OpaqueTy(..) => Err("opaque type"), // FIXME should implement this (e.g., pub use). diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml index cdff1662fdb..4c72920502f 100644 --- a/compiler/rustc_session/Cargo.toml +++ b/compiler/rustc_session/Cargo.toml @@ -18,3 +18,4 @@ rustc_span = { path = "../rustc_span" } rustc_fs_util = { path = "../rustc_fs_util" } num_cpus = "1.0" rustc_ast = { path = "../rustc_ast" } +rustc_lint_defs = { path = "../rustc_lint_defs" } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index f33bebf99d6..b632bfbed30 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2057,10 +2057,7 @@ impl PpMode { pub fn needs_analysis(&self) -> bool { use PpMode::*; - match *self { - PpmMir | PpmMirCFG => true, - _ => false, - } + matches!(*self, PpmMir | PpmMirCFG) } } diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs index 12a268d5b1d..55ee4e52082 100644 --- a/compiler/rustc_session/src/filesearch.rs +++ b/compiler/rustc_session/src/filesearch.rs @@ -153,14 +153,14 @@ fn find_libdir(sysroot: &Path) -> Cow<'static, str> { const SECONDARY_LIB_DIR: &str = "lib"; match option_env!("CFG_LIBDIR_RELATIVE") { - Some(libdir) if libdir != "lib" => libdir.into(), - _ => { + None | Some("lib") => { if sysroot.join(PRIMARY_LIB_DIR).join(RUST_LIB_DIR).exists() { PRIMARY_LIB_DIR.into() } else { SECONDARY_LIB_DIR.into() } } + Some(libdir) => libdir.into(), } } diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs index a808261798d..d002f597391 100644 --- a/compiler/rustc_session/src/lib.rs +++ b/compiler/rustc_session/src/lib.rs @@ -9,8 +9,8 @@ extern crate rustc_macros; pub mod cgu_reuse_tracker; pub mod utils; -#[macro_use] -pub mod lint; +pub use lint::{declare_lint, declare_lint_pass, declare_tool_lint, impl_lint_pass}; +pub use rustc_lint_defs as lint; pub mod parse; mod code_stats; diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 627adcceb3f..fb09773fd1b 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -717,7 +717,7 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options, // This list is in alphabetical order. // // If you add a new option, please update: - // - src/librustc_interface/tests.rs + // - compiler/rustc_interface/src/tests.rs // - src/doc/rustc/src/codegen-options/index.md ar: String = (String::new(), parse_string, [UNTRACKED], @@ -814,7 +814,7 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options, // This list is in alphabetical order. // // If you add a new option, please update: - // - src/librustc_interface/tests.rs + // - compiler/rustc_interface/src/tests.rs // - src/doc/rustc/src/codegen-options/index.md } @@ -825,7 +825,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, // This list is in alphabetical order. // // If you add a new option, please update: - // - src/librustc_interface/tests.rs + // - compiler/rustc_interface/src/tests.rs allow_features: Option<Vec<String>> = (None, parse_opt_comma_list, [TRACKED], "only allow the listed language features to be enabled in code (space separated)"), @@ -893,6 +893,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, all `statement`s (including terminators), only `terminator` spans, or \ computed `block` spans (one span encompassing a block's terminator and \ all statements)."), + emit_future_incompat_report: bool = (false, parse_bool, [UNTRACKED], + "emits a future-incompatibility report for lints (RFC 2834)"), emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED], "emit a section containing stack size metadata (default: no)"), fewer_names: bool = (false, parse_bool, [TRACKED], @@ -904,6 +906,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "force all crates to be `rustc_private` unstable (default: no)"), fuel: Option<(String, u64)> = (None, parse_optimization_fuel, [TRACKED], "set the optimization fuel quota for a crate"), + function_sections: Option<bool> = (None, parse_opt_bool, [TRACKED], + "whether each function should go in its own section"), graphviz_dark_mode: bool = (false, parse_bool, [UNTRACKED], "use dark-themed colors in graphviz output (default: no)"), graphviz_font: String = ("Courier, monospace".to_string(), parse_string, [UNTRACKED], @@ -1028,6 +1032,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "enable queries of the dependency graph for regression testing (default: no)"), query_stats: bool = (false, parse_bool, [UNTRACKED], "print some statistics about the query system (default: no)"), + relax_elf_relocations: Option<bool> = (None, parse_opt_bool, [TRACKED], + "whether ELF relocations can be relaxed"), relro_level: Option<RelroLevel> = (None, parse_relro_level, [TRACKED], "choose which RELRO level to use"), report_delayed_bugs: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_session/src/output.rs b/compiler/rustc_session/src/output.rs index 0766c55da74..130c3a06122 100644 --- a/compiler/rustc_session/src/output.rs +++ b/compiler/rustc_session/src/output.rs @@ -199,10 +199,8 @@ pub fn invalid_output_for_target(sess: &Session, crate_type: CrateType) -> bool _ => {} } } - if !sess.target.options.executables { - if crate_type == CrateType::Executable { - return true; - } + if !sess.target.options.executables && crate_type == CrateType::Executable { + return true; } false diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 8312f89b271..0b7c35a8afd 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -3,7 +3,7 @@ use crate::code_stats::CodeStats; pub use crate::code_stats::{DataTypeKind, FieldInfo, SizeKind, VariantInfo}; use crate::config::{self, CrateType, OutputType, PrintRequest, SanitizerSet, SwitchWithOptPath}; use crate::filesearch; -use crate::lint; +use crate::lint::{self, LintId}; use crate::parse::ParseSess; use crate::search_paths::{PathKind, SearchPath}; @@ -21,7 +21,8 @@ use rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitterWriter; use rustc_errors::emitter::{Emitter, EmitterWriter, HumanReadableErrorType}; use rustc_errors::json::JsonEmitter; use rustc_errors::registry::Registry; -use rustc_errors::{Applicability, DiagnosticBuilder, DiagnosticId, ErrorReported}; +use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, DiagnosticId, ErrorReported}; +use rustc_lint_defs::FutureBreakage; use rustc_span::edition::Edition; use rustc_span::source_map::{FileLoader, MultiSpan, RealFileLoader, SourceMap, Span}; use rustc_span::{sym, SourceFileHashAlgorithm, Symbol}; @@ -40,6 +41,10 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +pub trait SessionLintStore: sync::Send + sync::Sync { + fn name_to_lint(&self, lint_name: &str) -> LintId; +} + pub struct OptimizationFuel { /// If `-zfuel=crate=n` is specified, initially set to `n`, otherwise `0`. remaining: u64, @@ -131,6 +136,8 @@ pub struct Session { features: OnceCell<rustc_feature::Features>, + lint_store: OnceCell<Lrc<dyn SessionLintStore>>, + /// The maximum recursion limit for potentially infinitely recursive /// operations such as auto-dereference and monomorphization. pub recursion_limit: OnceCell<Limit>, @@ -297,6 +304,35 @@ impl Session { pub fn finish_diagnostics(&self, registry: &Registry) { self.check_miri_unleashed_features(); self.diagnostic().print_error_count(registry); + self.emit_future_breakage(); + } + + fn emit_future_breakage(&self) { + if !self.opts.debugging_opts.emit_future_incompat_report { + return; + } + + let diags = self.diagnostic().take_future_breakage_diagnostics(); + if diags.is_empty() { + return; + } + // If any future-breakage lints were registered, this lint store + // should be available + let lint_store = self.lint_store.get().expect("`lint_store` not initialized!"); + let diags_and_breakage: Vec<(FutureBreakage, Diagnostic)> = diags + .into_iter() + .map(|diag| { + let lint_name = match &diag.code { + Some(DiagnosticId::Lint { name, has_future_breakage: true }) => name, + _ => panic!("Unexpected code in diagnostic {:?}", diag), + }; + let lint = lint_store.name_to_lint(&lint_name); + let future_breakage = + lint.lint.future_incompatible.unwrap().future_breakage.unwrap(); + (future_breakage, diag) + }) + .collect(); + self.parse_sess.span_diagnostic.emit_future_breakage_report(diags_and_breakage); } pub fn local_crate_disambiguator(&self) -> CrateDisambiguator { @@ -337,6 +373,12 @@ impl Session { pub fn struct_warn(&self, msg: &str) -> DiagnosticBuilder<'_> { self.diagnostic().struct_warn(msg) } + pub fn struct_span_allow<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'_> { + self.diagnostic().struct_span_allow(sp, msg) + } + pub fn struct_allow(&self, msg: &str) -> DiagnosticBuilder<'_> { + self.diagnostic().struct_allow(msg) + } pub fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, msg: &str) -> DiagnosticBuilder<'_> { self.diagnostic().struct_span_err(sp, msg) } @@ -611,6 +653,13 @@ impl Session { } } + pub fn init_lint_store(&self, lint_store: Lrc<dyn SessionLintStore>) { + self.lint_store + .set(lint_store) + .map_err(|_| ()) + .expect("`lint_store` was initialized twice"); + } + /// Calculates the flavor of LTO to use for this compilation. pub fn lto(&self) -> config::Lto { // If our target has codegen requirements ignore the command line @@ -1388,6 +1437,7 @@ pub fn build_session( crate_types: OnceCell::new(), crate_disambiguator: OnceCell::new(), features: OnceCell::new(), + lint_store: OnceCell::new(), recursion_limit: OnceCell::new(), type_length_limit: OnceCell::new(), const_eval_limit: OnceCell::new(), diff --git a/compiler/rustc_span/src/caching_source_map_view.rs b/compiler/rustc_span/src/caching_source_map_view.rs index 68b0bd1a574..15dd00fb483 100644 --- a/compiler/rustc_span/src/caching_source_map_view.rs +++ b/compiler/rustc_span/src/caching_source_map_view.rs @@ -1,13 +1,25 @@ use crate::source_map::SourceMap; use crate::{BytePos, SourceFile}; use rustc_data_structures::sync::Lrc; +use std::ops::Range; #[derive(Clone)] struct CacheEntry { time_stamp: usize, line_number: usize, - line_start: BytePos, - line_end: BytePos, + // The line's byte position range in the `SourceMap`. This range will fail to contain a valid + // position in certain edge cases. Spans often start/end one past something, and when that + // something is the last character of a file (this can happen when a file doesn't end in a + // newline, for example), we'd still like for the position to be considered within the last + // line. However, it isn't according to the exclusive upper bound of this range. We cannot + // change the upper bound to be inclusive, because for most lines, the upper bound is the same + // as the lower bound of the next line, so there would be an ambiguity. + // + // Since the containment aspect of this range is only used to see whether or not the cache + // entry contains a position, the only ramification of the above is that we will get cache + // misses for these rare positions. A line lookup for the position via `SourceMap::lookup_line` + // after a cache miss will produce the last line number, as desired. + line: Range<BytePos>, file: Lrc<SourceFile>, file_index: usize, } @@ -26,8 +38,7 @@ impl<'sm> CachingSourceMapView<'sm> { let entry = CacheEntry { time_stamp: 0, line_number: 0, - line_start: BytePos(0), - line_end: BytePos(0), + line: BytePos(0)..BytePos(0), file: first_file, file_index: 0, }; @@ -47,13 +58,13 @@ impl<'sm> CachingSourceMapView<'sm> { // Check if the position is in one of the cached lines for cache_entry in self.line_cache.iter_mut() { - if pos >= cache_entry.line_start && pos < cache_entry.line_end { + if cache_entry.line.contains(&pos) { cache_entry.time_stamp = self.time_stamp; return Some(( cache_entry.file.clone(), cache_entry.line_number, - pos - cache_entry.line_start, + pos - cache_entry.line.start, )); } } @@ -69,13 +80,13 @@ impl<'sm> CachingSourceMapView<'sm> { let cache_entry = &mut self.line_cache[oldest]; // If the entry doesn't point to the correct file, fix it up - if pos < cache_entry.file.start_pos || pos >= cache_entry.file.end_pos { + if !file_contains(&cache_entry.file, pos) { let file_valid; if self.source_map.files().len() > 0 { let file_index = self.source_map.lookup_source_file_idx(pos); let file = self.source_map.files()[file_index].clone(); - if pos >= file.start_pos && pos < file.end_pos { + if file_contains(&file, pos) { cache_entry.file = file; cache_entry.file_index = file_index; file_valid = true; @@ -95,10 +106,19 @@ impl<'sm> CachingSourceMapView<'sm> { let line_bounds = cache_entry.file.line_bounds(line_index); cache_entry.line_number = line_index + 1; - cache_entry.line_start = line_bounds.0; - cache_entry.line_end = line_bounds.1; + cache_entry.line = line_bounds; cache_entry.time_stamp = self.time_stamp; - Some((cache_entry.file.clone(), cache_entry.line_number, pos - cache_entry.line_start)) + Some((cache_entry.file.clone(), cache_entry.line_number, pos - cache_entry.line.start)) } } + +#[inline] +fn file_contains(file: &SourceFile, pos: BytePos) -> bool { + // `SourceMap::lookup_source_file_idx` and `SourceFile::contains` both consider the position + // one past the end of a file to belong to it. Normally, that's what we want. But for the + // purposes of converting a byte position to a line and column number, we can't come up with a + // line and column number if the file is empty, because an empty file doesn't contain any + // lines. So for our purposes, we don't consider empty files to contain any byte position. + file.contains(pos) && !file.is_empty() +} diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs index d036c078049..0e3027273ab 100644 --- a/compiler/rustc_span/src/lib.rs +++ b/compiler/rustc_span/src/lib.rs @@ -52,7 +52,7 @@ use std::cell::RefCell; use std::cmp::{self, Ordering}; use std::fmt; use std::hash::Hash; -use std::ops::{Add, Sub}; +use std::ops::{Add, Range, Sub}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -738,14 +738,14 @@ impl<D: Decoder> Decodable<D> for Span { } /// Calls the provided closure, using the provided `SourceMap` to format -/// any spans that are debug-printed during the closure'e exectuino. +/// any spans that are debug-printed during the closure's execution. /// /// Normally, the global `TyCtxt` is used to retrieve the `SourceMap` /// (see `rustc_interface::callbacks::span_debug1). However, some parts /// of the compiler (e.g. `rustc_parse`) may debug-print `Span`s before /// a `TyCtxt` is available. In this case, we fall back to /// the `SourceMap` provided to this function. If that is not available, -/// we fall back to printing the raw `Span` field values +/// we fall back to printing the raw `Span` field values. pub fn with_source_map<T, F: FnOnce() -> T>(source_map: Lrc<SourceMap>, f: F) -> T { SESSION_GLOBALS.with(|session_globals| { *session_globals.source_map.borrow_mut() = Some(source_map); @@ -1426,24 +1426,33 @@ impl SourceFile { if line_index >= 0 { Some(line_index as usize) } else { None } } - pub fn line_bounds(&self, line_index: usize) -> (BytePos, BytePos) { - if self.start_pos == self.end_pos { - return (self.start_pos, self.end_pos); + pub fn line_bounds(&self, line_index: usize) -> Range<BytePos> { + if self.is_empty() { + return self.start_pos..self.end_pos; } assert!(line_index < self.lines.len()); if line_index == (self.lines.len() - 1) { - (self.lines[line_index], self.end_pos) + self.lines[line_index]..self.end_pos } else { - (self.lines[line_index], self.lines[line_index + 1]) + self.lines[line_index]..self.lines[line_index + 1] } } + /// Returns whether or not the file contains the given `SourceMap` byte + /// position. The position one past the end of the file is considered to be + /// contained by the file. This implies that files for which `is_empty` + /// returns true still contain one byte position according to this function. #[inline] pub fn contains(&self, byte_pos: BytePos) -> bool { byte_pos >= self.start_pos && byte_pos <= self.end_pos } + #[inline] + pub fn is_empty(&self) -> bool { + self.start_pos == self.end_pos + } + /// Calculates the original byte position relative to the start of the file /// based on the given byte position. pub fn original_relative_byte_pos(&self, pos: BytePos) -> BytePos { @@ -1925,9 +1934,7 @@ impl<CTX: HashStableContext> HashStable<CTX> for ExpnId { return; } - TAG_NOT_ROOT.hash_stable(ctx, hasher); let index = self.as_u32() as usize; - let res = CACHE.with(|cache| cache.borrow().get(index).copied().flatten()); if let Some(res) = res { @@ -1936,6 +1943,7 @@ impl<CTX: HashStableContext> HashStable<CTX> for ExpnId { let new_len = index + 1; let mut sub_hasher = StableHasher::new(); + TAG_NOT_ROOT.hash_stable(ctx, &mut sub_hasher); self.expn_data().hash_stable(ctx, &mut sub_hasher); let sub_hash: Fingerprint = sub_hasher.finish(); diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs index 3b929c4acb9..f067cdb7308 100644 --- a/compiler/rustc_span/src/source_map.rs +++ b/compiler/rustc_span/src/source_map.rs @@ -12,7 +12,7 @@ pub use crate::*; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::StableHasher; -use rustc_data_structures::sync::{AtomicU32, Lock, LockGuard, Lrc, MappedLockGuard}; +use rustc_data_structures::sync::{AtomicU32, Lrc, MappedReadGuard, ReadGuard, RwLock}; use std::cmp; use std::convert::TryFrom; use std::hash::Hash; @@ -168,7 +168,7 @@ pub struct SourceMap { /// The address space below this value is currently used by the files in the source map. used_address_space: AtomicU32, - files: Lock<SourceMapFiles>, + files: RwLock<SourceMapFiles>, file_loader: Box<dyn FileLoader + Sync + Send>, // This is used to apply the file path remapping as specified via // `--remap-path-prefix` to all `SourceFile`s allocated within this `SourceMap`. @@ -236,8 +236,8 @@ impl SourceMap { // By returning a `MonotonicVec`, we ensure that consumers cannot invalidate // any existing indices pointing into `files`. - pub fn files(&self) -> MappedLockGuard<'_, monotonic::MonotonicVec<Lrc<SourceFile>>> { - LockGuard::map(self.files.borrow(), |files| &mut files.source_files) + pub fn files(&self) -> MappedReadGuard<'_, monotonic::MonotonicVec<Lrc<SourceFile>>> { + ReadGuard::map(self.files.borrow(), |files| &files.source_files) } pub fn source_file_by_stable_id( diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 9cf530d57c0..1a6c45b6c80 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -127,6 +127,7 @@ symbols! { ArgumentV1, Arguments, C, + CString, Center, Clone, Copy, @@ -261,6 +262,7 @@ symbols! { arm_target_feature, array, arrays, + as_ptr, as_str, asm, assert, @@ -310,6 +312,7 @@ symbols! { breakpoint, bridge, bswap, + c_str, c_variadic, call, call_mut, @@ -397,6 +400,7 @@ symbols! { crate_type, crate_visibility_modifier, crt_dash_static: "crt-static", + cstring_type, ctlz, ctlz_nonzero, ctpop, @@ -467,6 +471,7 @@ symbols! { encode, env, eq, + ermsb_target_feature, err, exact_div, except, @@ -477,6 +482,7 @@ symbols! { existential_type, exp2f32, exp2f64, + expect, expected, expf32, expf64, @@ -500,6 +506,7 @@ symbols! { fadd_fast, fdiv_fast, feature, + ffi, ffi_const, ffi_pure, ffi_returns_twice, @@ -777,6 +784,7 @@ symbols! { panic_info, panic_location, panic_runtime, + panic_str, panic_unwind, param_attrs, parent_trait, @@ -794,6 +802,8 @@ symbols! { plugin_registrar, plugins, pointer, + pointer_trait, + pointer_trait_fmt, poll, position, post_dash_lto: "post-lto", @@ -893,6 +903,7 @@ symbols! { rustc, rustc_allocator, rustc_allocator_nounwind, + rustc_allow_const_fn_unstable, rustc_args_required_const, rustc_attrs, rustc_builtin_macro, @@ -1158,6 +1169,7 @@ symbols! { unsafe_cell, unsafe_no_drop_flag, unsize, + unsized_fn_params, unsized_locals, unsized_tuple_coercion, unstable, @@ -1165,6 +1177,7 @@ symbols! { unused_qualifications, unwind, unwind_attributes, + unwrap, unwrap_or, use_extern_macros, use_nested_groups, diff --git a/compiler/rustc_symbol_mangling/Cargo.toml b/compiler/rustc_symbol_mangling/Cargo.toml index c0dacd24c38..3df5f161319 100644 --- a/compiler/rustc_symbol_mangling/Cargo.toml +++ b/compiler/rustc_symbol_mangling/Cargo.toml @@ -10,7 +10,7 @@ doctest = false [dependencies] tracing = "0.1" punycode = "0.4.0" -rustc-demangle = "0.1.16" +rustc-demangle = "0.1.18" rustc_ast = { path = "../rustc_ast" } rustc_span = { path = "../rustc_span" } diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs index 2c9caf73b8e..e8d470297ea 100644 --- a/compiler/rustc_symbol_mangling/src/legacy.rs +++ b/compiler/rustc_symbol_mangling/src/legacy.rs @@ -326,10 +326,8 @@ impl Printer<'tcx> for SymbolPrinter<'tcx> { ) -> Result<Self::Path, Self::Error> { self = print_prefix(self)?; - let args = args.iter().cloned().filter(|arg| match arg.unpack() { - GenericArgKind::Lifetime(_) => false, - _ => true, - }); + let args = + args.iter().cloned().filter(|arg| !matches!(arg.unpack(), GenericArgKind::Lifetime(_))); if args.clone().next().is_some() { self.generic_delimiters(|cx| cx.comma_sep(args)) diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs index 28b4a78929e..10245d21b63 100644 --- a/compiler/rustc_symbol_mangling/src/lib.rs +++ b/compiler/rustc_symbol_mangling/src/lib.rs @@ -174,10 +174,7 @@ fn compute_symbol_name( return tcx.sess.generate_proc_macro_decls_symbol(disambiguator); } let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - match tcx.hir().get(hir_id) { - Node::ForeignItem(_) => true, - _ => false, - } + matches!(tcx.hir().get(hir_id), Node::ForeignItem(_)) } else { tcx.is_foreign_item(def_id) }; @@ -200,15 +197,14 @@ fn compute_symbol_name( // show up in the `wasm-import-name` custom attribute in LLVM IR. // // [1]: https://bugs.llvm.org/show_bug.cgi?id=44316 - if is_foreign { - if tcx.sess.target.arch != "wasm32" - || !tcx.wasm_import_module_map(def_id.krate).contains_key(&def_id) - { - if let Some(name) = attrs.link_name { - return name.to_string(); - } - return tcx.item_name(def_id).to_string(); + if is_foreign + && (tcx.sess.target.arch != "wasm32" + || !tcx.wasm_import_module_map(def_id.krate).contains_key(&def_id)) + { + if let Some(name) = attrs.link_name { + return name.to_string(); } + return tcx.item_name(def_id).to_string(); } if let Some(name) = attrs.export_name { @@ -234,10 +230,7 @@ fn compute_symbol_name( // codegen units) then this symbol may become an exported (but hidden // visibility) symbol. This means that multiple crates may do the same // and we want to be sure to avoid any symbol conflicts here. - match MonoItem::Fn(instance).instantiation_mode(tcx) { - InstantiationMode::GloballyShared { may_conflict: true } => true, - _ => false, - }; + matches!(MonoItem::Fn(instance).instantiation_mode(tcx), InstantiationMode::GloballyShared { may_conflict: true }); let instantiating_crate = if avoid_cross_crate_conflicts { Some(compute_instantiating_crate()) } else { None }; diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs index 16d0b86903e..1ff043ae91f 100644 --- a/compiler/rustc_symbol_mangling/src/v0.rs +++ b/compiler/rustc_symbol_mangling/src/v0.rs @@ -4,6 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir as hir; use rustc_hir::def_id::{CrateNum, DefId}; use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData}; +use rustc_middle::mir::interpret::sign_extend; use rustc_middle::ty::print::{Print, Printer}; use rustc_middle::ty::subst::{GenericArg, GenericArgKind, Subst}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable}; @@ -199,15 +200,9 @@ impl SymbolMangler<'tcx> { let lifetimes = regions .into_iter() - .map(|br| { - match br { - ty::BrAnon(i) => { - // FIXME(eddyb) for some reason, `anonymize_late_bound_regions` starts at `1`. - assert_ne!(i, 0); - i - 1 - } - _ => bug!("symbol_names: non-anonymized region `{:?}` in `{:?}`", br, value), - } + .map(|br| match br { + ty::BrAnon(i) => i, + _ => bug!("symbol_names: non-anonymized region `{:?}` in `{:?}`", br, value), }) .max() .map_or(0, |max| max + 1); @@ -326,10 +321,6 @@ impl Printer<'tcx> for SymbolMangler<'tcx> { // Late-bound lifetimes use indices starting at 1, // see `BinderLevel` for more details. ty::ReLateBound(debruijn, ty::BrAnon(i)) => { - // FIXME(eddyb) for some reason, `anonymize_late_bound_regions` starts at `1`. - assert_ne!(i, 0); - let i = i - 1; - let binder = &self.binders[self.binders.len() - 1 - debruijn.index()]; let depth = binder.lifetime_depths.start + i; @@ -527,17 +518,31 @@ impl Printer<'tcx> for SymbolMangler<'tcx> { } let start = self.out.len(); - match ct.ty.kind() { - ty::Uint(_) => {} - ty::Bool => {} + let mut neg = false; + let val = match ct.ty.kind() { + ty::Uint(_) | ty::Bool | ty::Char => { + ct.try_eval_bits(self.tcx, ty::ParamEnv::reveal_all(), ct.ty) + } + ty::Int(_) => { + let param_env = ty::ParamEnv::reveal_all(); + ct.try_eval_bits(self.tcx, param_env, ct.ty).and_then(|b| { + let sz = self.tcx.layout_of(param_env.and(ct.ty)).ok()?.size; + let val = sign_extend(b, sz) as i128; + if val < 0 { + neg = true; + } + Some(val.wrapping_abs() as u128) + }) + } _ => { bug!("symbol_names: unsupported constant of type `{}` ({:?})", ct.ty, ct); } - } - self = ct.ty.print(self)?; + }; - if let Some(bits) = ct.try_eval_bits(self.tcx, ty::ParamEnv::reveal_all(), ct.ty) { - let _ = write!(self.out, "{:x}_", bits); + if let Some(bits) = val { + // We only print the type if the const can be evaluated. + self = ct.ty.print(self)?; + let _ = write!(self.out, "{}{:x}_", if neg { "n" } else { "" }, bits); } else { // NOTE(eddyb) despite having the path, we need to // encode a placeholder, as the path could refer diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 8f7e2bba5aa..507ae10877b 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -474,31 +474,19 @@ impl<'a, Ty> ArgAbi<'a, Ty> { } pub fn is_indirect(&self) -> bool { - match self.mode { - PassMode::Indirect(..) => true, - _ => false, - } + matches!(self.mode, PassMode::Indirect(..)) } pub fn is_sized_indirect(&self) -> bool { - match self.mode { - PassMode::Indirect(_, None) => true, - _ => false, - } + matches!(self.mode, PassMode::Indirect(_, None)) } pub fn is_unsized_indirect(&self) -> bool { - match self.mode { - PassMode::Indirect(_, Some(_)) => true, - _ => false, - } + matches!(self.mode, PassMode::Indirect(_, Some(_))) } pub fn is_ignore(&self) -> bool { - match self.mode { - PassMode::Ignore => true, - _ => false, - } + matches!(self.mode, PassMode::Ignore) } } diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs index 2e10bed3bd4..47530eeacd0 100644 --- a/compiler/rustc_target/src/abi/call/riscv.rs +++ b/compiler/rustc_target/src/abi/call/riscv.rs @@ -333,10 +333,8 @@ where let mut avail_gprs = 8; let mut avail_fprs = 8; - if !fn_abi.ret.is_ignore() { - if classify_ret(cx, &mut fn_abi.ret, xlen, flen) { - avail_gprs -= 1; - } + if !fn_abi.ret.is_ignore() && classify_ret(cx, &mut fn_abi.ret, xlen, flen) { + avail_gprs -= 1; } for (i, arg) in fn_abi.args.iter_mut().enumerate() { diff --git a/compiler/rustc_target/src/abi/call/wasm32.rs b/compiler/rustc_target/src/abi/call/wasm32.rs index 510f671a501..ff2c0e9bb6f 100644 --- a/compiler/rustc_target/src/abi/call/wasm32.rs +++ b/compiler/rustc_target/src/abi/call/wasm32.rs @@ -24,10 +24,8 @@ where C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout, { ret.extend_integer_width_to(32); - if ret.layout.is_aggregate() { - if !unwrap_trivial_aggregate(cx, ret) { - ret.make_indirect(); - } + if ret.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, ret) { + ret.make_indirect(); } } @@ -37,10 +35,8 @@ where C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout, { arg.extend_integer_width_to(32); - if arg.layout.is_aggregate() { - if !unwrap_trivial_aggregate(cx, arg) { - arg.make_indirect_byval(); - } + if arg.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, arg) { + arg.make_indirect_byval(); } } diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index 047b8cf5cdb..5c87a8f6b3d 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -557,17 +557,11 @@ impl Primitive { } pub fn is_float(self) -> bool { - match self { - F32 | F64 => true, - _ => false, - } + matches!(self, F32 | F64) } pub fn is_int(self) -> bool { - match self { - Int(..) => true, - _ => false, - } + matches!(self, Int(..)) } } @@ -794,18 +788,12 @@ impl Abi { /// Returns `true` if this is an uninhabited type pub fn is_uninhabited(&self) -> bool { - match *self { - Abi::Uninhabited => true, - _ => false, - } + matches!(*self, Abi::Uninhabited) } /// Returns `true` is this is a scalar type pub fn is_scalar(&self) -> bool { - match *self { - Abi::Scalar(_) => true, - _ => false, - } + matches!(*self, Abi::Scalar(_)) } } diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index 0d691dc441e..8e60085262a 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -478,10 +478,7 @@ pub enum InlineAsmType { impl InlineAsmType { pub fn is_integer(self) -> bool { - match self { - Self::I8 | Self::I16 | Self::I32 | Self::I64 | Self::I128 => true, - _ => false, - } + matches!(self, Self::I8 | Self::I16 | Self::I32 | Self::I64 | Self::I128) } pub fn size(self) -> Size { diff --git a/compiler/rustc_target/src/spec/apple_sdk_base.rs b/compiler/rustc_target/src/spec/apple_sdk_base.rs index e34277d5af0..1b17c2c278f 100644 --- a/compiler/rustc_target/src/spec/apple_sdk_base.rs +++ b/compiler/rustc_target/src/spec/apple_sdk_base.rs @@ -34,6 +34,7 @@ fn link_env_remove(arch: Arch) -> Vec<String> { pub fn opts(arch: Arch) -> TargetOptions { TargetOptions { cpu: target_cpu(arch), + dynamic_linking: false, executables: true, link_env_remove: link_env_remove(arch), has_elf_tls: false, diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs index 406e8936e6e..42509cd8975 100644 --- a/compiler/rustc_trait_selection/src/lib.rs +++ b/compiler/rustc_trait_selection/src/lib.rs @@ -19,6 +19,7 @@ #![feature(never_type)] #![feature(crate_visibility_modifier)] #![feature(or_patterns)] +#![feature(control_flow_enum)] #![recursion_limit = "512"] // For rustdoc #[macro_use] diff --git a/compiler/rustc_trait_selection/src/opaque_types.rs b/compiler/rustc_trait_selection/src/opaque_types.rs index ecaafee77e2..914fa1e52c2 100644 --- a/compiler/rustc_trait_selection/src/opaque_types.rs +++ b/compiler/rustc_trait_selection/src/opaque_types.rs @@ -15,6 +15,8 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::config::nightly_options; use rustc_span::Span; +use std::ops::ControlFlow; + pub type OpaqueTypeMap<'tcx> = DefIdMap<OpaqueTypeDecl<'tcx>>; /// Information about the opaque types whose values we @@ -691,26 +693,26 @@ impl<'tcx, OP> TypeVisitor<'tcx> for ConstrainOpaqueTypeRegionVisitor<OP> where OP: FnMut(ty::Region<'tcx>), { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &ty::Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &ty::Binder<T>) -> ControlFlow<()> { t.as_ref().skip_binder().visit_with(self); - false // keep visiting + ControlFlow::CONTINUE } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { match *r { // ignore bound regions, keep visiting - ty::ReLateBound(_, _) => false, + ty::ReLateBound(_, _) => ControlFlow::CONTINUE, _ => { (self.op)(r); - false + ControlFlow::CONTINUE } } } - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { // We're only interested in types involving regions if !ty.flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS) { - return false; // keep visiting + return ControlFlow::CONTINUE; } match ty.kind() { @@ -745,7 +747,7 @@ where } } - false + ControlFlow::CONTINUE } } diff --git a/compiler/rustc_trait_selection/src/traits/codegen/mod.rs b/compiler/rustc_trait_selection/src/traits/codegen.rs index 05e6c4804ff..3cb6ec86261 100644 --- a/compiler/rustc_trait_selection/src/traits/codegen/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/codegen.rs @@ -121,7 +121,10 @@ where // contains unbound type parameters. It could be a slight // optimization to stop iterating early. if let Err(errors) = fulfill_cx.select_all_or_error(infcx) { - bug!("Encountered errors `{:?}` resolving bounds after type-checking", errors); + infcx.tcx.sess.delay_span_bug( + rustc_span::DUMMY_SP, + &format!("Encountered errors `{:?}` resolving bounds after type-checking", errors), + ); } let result = infcx.resolve_vars_if_possible(result); diff --git a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs index 1e1eb16faf4..638a8253e7e 100644 --- a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs +++ b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs @@ -24,6 +24,7 @@ use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::Span; use std::cmp; +use std::ops::ControlFlow; /// Check if a given constant can be evaluated. pub fn is_const_evaluatable<'cx, 'tcx>( @@ -85,8 +86,12 @@ pub fn is_const_evaluatable<'cx, 'tcx>( } else if leaf.has_param_types_or_consts() { failure_kind = cmp::min(failure_kind, FailureKind::MentionsParam); } + + ControlFlow::CONTINUE + } + Node::Binop(_, _, _) | Node::UnaryOp(_, _) | Node::FunctionCall(_, _) => { + ControlFlow::CONTINUE } - Node::Binop(_, _, _) | Node::UnaryOp(_, _) | Node::FunctionCall(_, _) => (), }); match failure_kind { @@ -194,12 +199,12 @@ pub fn is_const_evaluatable<'cx, 'tcx>( /// /// This is only able to represent a subset of `MIR`, /// and should not leak any information about desugarings. -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub struct AbstractConst<'tcx> { // FIXME: Consider adding something like `IndexSlice` // and use this here. - inner: &'tcx [Node<'tcx>], - substs: SubstsRef<'tcx>, + pub inner: &'tcx [Node<'tcx>], + pub substs: SubstsRef<'tcx>, } impl AbstractConst<'tcx> { @@ -209,9 +214,21 @@ impl AbstractConst<'tcx> { substs: SubstsRef<'tcx>, ) -> Result<Option<AbstractConst<'tcx>>, ErrorReported> { let inner = tcx.mir_abstract_const_opt_const_arg(def)?; + debug!("AbstractConst::new({:?}) = {:?}", def, inner); Ok(inner.map(|inner| AbstractConst { inner, substs })) } + pub fn from_const( + tcx: TyCtxt<'tcx>, + ct: &ty::Const<'tcx>, + ) -> Result<Option<AbstractConst<'tcx>>, ErrorReported> { + match ct.val { + ty::ConstKind::Unevaluated(def, substs, None) => AbstractConst::new(tcx, def, substs), + ty::ConstKind::Error(_) => Err(ErrorReported), + _ => Ok(None), + } + } + #[inline] pub fn subtree(self, node: NodeId) -> AbstractConst<'tcx> { AbstractConst { inner: &self.inner[..=node.index()], substs: self.substs } @@ -223,11 +240,23 @@ impl AbstractConst<'tcx> { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct WorkNode<'tcx> { + node: Node<'tcx>, + span: Span, + used: bool, +} + struct AbstractConstBuilder<'a, 'tcx> { tcx: TyCtxt<'tcx>, body: &'a mir::Body<'tcx>, /// The current WIP node tree. - nodes: IndexVec<NodeId, Node<'tcx>>, + /// + /// We require all nodes to be used in the final abstract const, + /// so we store this here. Note that we also consider nodes as used + /// if they are mentioned in an assert, so some used nodes are never + /// actually reachable by walking the [`AbstractConst`]. + nodes: IndexVec<NodeId, WorkNode<'tcx>>, locals: IndexVec<mir::Local, NodeId>, /// We only allow field accesses if they access /// the result of a checked operation. @@ -274,6 +303,27 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { Ok(Some(builder)) } + fn add_node(&mut self, node: Node<'tcx>, span: Span) -> NodeId { + // Mark used nodes. + match node { + Node::Leaf(_) => (), + Node::Binop(_, lhs, rhs) => { + self.nodes[lhs].used = true; + self.nodes[rhs].used = true; + } + Node::UnaryOp(_, input) => { + self.nodes[input].used = true; + } + Node::FunctionCall(func, nodes) => { + self.nodes[func].used = true; + nodes.iter().for_each(|&n| self.nodes[n].used = true); + } + } + + // Nodes start as unused. + self.nodes.push(WorkNode { node, span, used: false }) + } + fn place_to_local( &mut self, span: Span, @@ -311,7 +361,7 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { let local = self.place_to_local(span, p)?; Ok(self.locals[local]) } - mir::Operand::Constant(ct) => Ok(self.nodes.push(Node::Leaf(ct.literal))), + mir::Operand::Constant(ct) => Ok(self.add_node(Node::Leaf(ct.literal), span)), } } @@ -336,19 +386,19 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { fn build_statement(&mut self, stmt: &mir::Statement<'tcx>) -> Result<(), ErrorReported> { debug!("AbstractConstBuilder: stmt={:?}", stmt); + let span = stmt.source_info.span; match stmt.kind { StatementKind::Assign(box (ref place, ref rvalue)) => { - let local = self.place_to_local(stmt.source_info.span, place)?; + let local = self.place_to_local(span, place)?; match *rvalue { Rvalue::Use(ref operand) => { - self.locals[local] = - self.operand_to_node(stmt.source_info.span, operand)?; + self.locals[local] = self.operand_to_node(span, operand)?; Ok(()) } Rvalue::BinaryOp(op, ref lhs, ref rhs) if Self::check_binop(op) => { - let lhs = self.operand_to_node(stmt.source_info.span, lhs)?; - let rhs = self.operand_to_node(stmt.source_info.span, rhs)?; - self.locals[local] = self.nodes.push(Node::Binop(op, lhs, rhs)); + let lhs = self.operand_to_node(span, lhs)?; + let rhs = self.operand_to_node(span, rhs)?; + self.locals[local] = self.add_node(Node::Binop(op, lhs, rhs), span); if op.is_checkable() { bug!("unexpected unchecked checkable binary operation"); } else { @@ -356,18 +406,18 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { } } Rvalue::CheckedBinaryOp(op, ref lhs, ref rhs) if Self::check_binop(op) => { - let lhs = self.operand_to_node(stmt.source_info.span, lhs)?; - let rhs = self.operand_to_node(stmt.source_info.span, rhs)?; - self.locals[local] = self.nodes.push(Node::Binop(op, lhs, rhs)); + let lhs = self.operand_to_node(span, lhs)?; + let rhs = self.operand_to_node(span, rhs)?; + self.locals[local] = self.add_node(Node::Binop(op, lhs, rhs), span); self.checked_op_locals.insert(local); Ok(()) } Rvalue::UnaryOp(op, ref operand) if Self::check_unop(op) => { - let operand = self.operand_to_node(stmt.source_info.span, operand)?; - self.locals[local] = self.nodes.push(Node::UnaryOp(op, operand)); + let operand = self.operand_to_node(span, operand)?; + self.locals[local] = self.add_node(Node::UnaryOp(op, operand), span); Ok(()) } - _ => self.error(Some(stmt.source_info.span), "unsupported rvalue")?, + _ => self.error(Some(span), "unsupported rvalue")?, } } // These are not actually relevant for us here, so we can ignore them. @@ -415,13 +465,9 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { .map(|arg| self.operand_to_node(terminator.source_info.span, arg)) .collect::<Result<Vec<NodeId>, _>>()?, ); - self.locals[local] = self.nodes.push(Node::FunctionCall(func, args)); + self.locals[local] = self.add_node(Node::FunctionCall(func, args), fn_span); Ok(Some(target)) } - // We only allow asserts for checked operations. - // - // These asserts seem to all have the form `!_local.0` so - // we only allow exactly that. TerminatorKind::Assert { ref cond, expected: false, target, .. } => { let p = match cond { mir::Operand::Copy(p) | mir::Operand::Move(p) => p, @@ -430,7 +476,15 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { const ONE_FIELD: mir::Field = mir::Field::from_usize(1); debug!("proj: {:?}", p.projection); - if let &[mir::ProjectionElem::Field(ONE_FIELD, _)] = p.projection.as_ref() { + if let Some(p) = p.as_local() { + debug_assert!(!self.checked_op_locals.contains(p)); + // Mark locals directly used in asserts as used. + // + // This is needed because division does not use `CheckedBinop` but instead + // adds an explicit assert for `divisor != 0`. + self.nodes[self.locals[p]].used = true; + return Ok(Some(target)); + } else if let &[mir::ProjectionElem::Field(ONE_FIELD, _)] = p.projection.as_ref() { // Only allow asserts checking the result of a checked operation. if self.checked_op_locals.contains(p.local) { return Ok(Some(target)); @@ -457,7 +511,13 @@ impl<'a, 'tcx> AbstractConstBuilder<'a, 'tcx> { if let Some(next) = self.build_terminator(block.terminator())? { block = &self.body.basic_blocks()[next]; } else { - return Ok(self.tcx.arena.alloc_from_iter(self.nodes)); + assert_eq!(self.locals[mir::RETURN_PLACE], self.nodes.last().unwrap()); + self.nodes[self.locals[mir::RETURN_PLACE]].used = true; + if let Some(&unused) = self.nodes.iter().find(|n| !n.used) { + self.error(Some(unused.span), "dead code")?; + } + + return Ok(self.tcx.arena.alloc_from_iter(self.nodes.into_iter().map(|n| n.node))); } } } @@ -507,31 +567,36 @@ pub(super) fn try_unify_abstract_consts<'tcx>( // on `ErrorReported`. } -fn walk_abstract_const<'tcx, F>(tcx: TyCtxt<'tcx>, ct: AbstractConst<'tcx>, mut f: F) +pub fn walk_abstract_const<'tcx, F>( + tcx: TyCtxt<'tcx>, + ct: AbstractConst<'tcx>, + mut f: F, +) -> ControlFlow<()> where - F: FnMut(Node<'tcx>), + F: FnMut(Node<'tcx>) -> ControlFlow<()>, { - recurse(tcx, ct, &mut f); - fn recurse<'tcx>(tcx: TyCtxt<'tcx>, ct: AbstractConst<'tcx>, f: &mut dyn FnMut(Node<'tcx>)) { + fn recurse<'tcx>( + tcx: TyCtxt<'tcx>, + ct: AbstractConst<'tcx>, + f: &mut dyn FnMut(Node<'tcx>) -> ControlFlow<()>, + ) -> ControlFlow<()> { let root = ct.root(); - f(root); + f(root)?; match root { - Node::Leaf(_) => (), + Node::Leaf(_) => ControlFlow::CONTINUE, Node::Binop(_, l, r) => { - recurse(tcx, ct.subtree(l), f); - recurse(tcx, ct.subtree(r), f); - } - Node::UnaryOp(_, v) => { - recurse(tcx, ct.subtree(v), f); + recurse(tcx, ct.subtree(l), f)?; + recurse(tcx, ct.subtree(r), f) } + Node::UnaryOp(_, v) => recurse(tcx, ct.subtree(v), f), Node::FunctionCall(func, args) => { - recurse(tcx, ct.subtree(func), f); - for &arg in args { - recurse(tcx, ct.subtree(arg), f); - } + recurse(tcx, ct.subtree(func), f)?; + args.iter().try_for_each(|&arg| recurse(tcx, ct.subtree(arg), f)) } } } + + recurse(tcx, ct, &mut f) } /// Tries to unify two abstract constants using structural equality. diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs index f53465266d2..f8bd3ab96e2 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs @@ -1462,9 +1462,8 @@ impl<'a, 'tcx> InferCtxtPrivExt<'tcx> for InferCtxt<'a, 'tcx> { let bound_predicate = predicate.bound_atom(); let mut err = match bound_predicate.skip_binder() { ty::PredicateAtom::Trait(data, _) => { - let self_ty = data.trait_ref.self_ty(); let trait_ref = bound_predicate.rebind(data.trait_ref); - debug!("self_ty {:?} {:?} trait_ref {:?}", self_ty, self_ty.kind(), trait_ref); + debug!("trait_ref {:?}", trait_ref); if predicate.references_error() { return; @@ -1479,6 +1478,17 @@ impl<'a, 'tcx> InferCtxtPrivExt<'tcx> for InferCtxt<'a, 'tcx> { // known, since we don't dispatch based on region // relationships. + // Pick the first substitution that still contains inference variables as the one + // we're going to emit an error for. If there are none (see above), fall back to + // the substitution for `Self`. + let subst = { + let substs = data.trait_ref.substs; + substs + .iter() + .find(|s| s.has_infer_types_or_consts()) + .unwrap_or_else(|| substs[0]) + }; + // This is kind of a hack: it frequently happens that some earlier // error prevents types from being fully inferred, and then we get // a bunch of uninteresting errors saying something like "<generic @@ -1495,21 +1505,11 @@ impl<'a, 'tcx> InferCtxtPrivExt<'tcx> for InferCtxt<'a, 'tcx> { // check upstream for type errors and don't add the obligations to // begin with in those cases. if self.tcx.lang_items().sized_trait() == Some(trait_ref.def_id()) { - self.emit_inference_failure_err( - body_id, - span, - self_ty.into(), - ErrorCode::E0282, - ) - .emit(); + self.emit_inference_failure_err(body_id, span, subst, ErrorCode::E0282).emit(); return; } - let mut err = self.emit_inference_failure_err( - body_id, - span, - self_ty.into(), - ErrorCode::E0283, - ); + let mut err = + self.emit_inference_failure_err(body_id, span, subst, ErrorCode::E0283); err.note(&format!("cannot satisfy `{}`", predicate)); if let ObligationCauseCode::ItemObligation(def_id) = obligation.cause.code { self.suggest_fully_qualified_path(&mut err, def_id, span, trait_ref.def_id()); diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs index efa9bd633ba..c0881befe24 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs @@ -21,7 +21,7 @@ use rustc_middle::ty::{ }; use rustc_middle::ty::{TypeAndMut, TypeckResults}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; -use rustc_span::{MultiSpan, Span, DUMMY_SP}; +use rustc_span::{BytePos, MultiSpan, Span, DUMMY_SP}; use rustc_target::spec::abi; use std::fmt; @@ -1845,9 +1845,9 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { err.note("all function arguments must have a statically known size"); } if tcx.sess.opts.unstable_features.is_nightly_build() - && !self.tcx.features().unsized_locals + && !self.tcx.features().unsized_fn_params { - err.help("unsized locals are gated as an unstable feature"); + err.help("unsized fn params are gated as an unstable feature"); } } ObligationCauseCode::SizedReturnType => { @@ -2114,10 +2114,10 @@ impl<'a, 'tcx> InferCtxtExt<'tcx> for InferCtxt<'a, 'tcx> { if self.predicate_may_hold(&try_obligation) && impls_future { if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span) { if snippet.ends_with('?') { - err.span_suggestion( - span, - "consider using `.await` here", - format!("{}.await?", snippet.trim_end_matches('?')), + err.span_suggestion_verbose( + span.with_hi(span.hi() - BytePos(1)).shrink_to_hi(), + "consider `await`ing on the `Future`", + ".await".to_string(), Applicability::MaybeIncorrect, ); } diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index 495a250be7c..9a8b5534dfe 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -1,6 +1,6 @@ use crate::infer::{InferCtxt, TyOrConstInferVar}; use rustc_data_structures::obligation_forest::ProcessResult; -use rustc_data_structures::obligation_forest::{DoCompleted, Error, ForestObligation}; +use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome}; use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor}; use rustc_errors::ErrorReported; use rustc_infer::traits::{TraitEngine, TraitEngineExt as _, TraitObligation}; @@ -129,13 +129,11 @@ impl<'a, 'tcx> FulfillmentContext<'tcx> { debug!("select: starting another iteration"); // Process pending obligations. - let outcome = self.predicates.process_obligations( - &mut FulfillProcessor { + let outcome: Outcome<_, _> = + self.predicates.process_obligations(&mut FulfillProcessor { selcx, register_region_obligations: self.register_region_obligations, - }, - DoCompleted::No, - ); + }); debug!("select: outcome={:#?}", outcome); // FIXME: if we kept the original cache key, we could mark projection diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs index 0e43f1655dd..50efbbbe0fd 100644 --- a/compiler/rustc_trait_selection/src/traits/object_safety.rs +++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs @@ -11,9 +11,10 @@ use super::elaborate_predicates; use crate::infer::TyCtxtInferExt; +use crate::traits::const_evaluatable::{self, AbstractConst}; use crate::traits::query::evaluate_obligation::InferCtxtExt; use crate::traits::{self, Obligation, ObligationCause}; -use rustc_errors::{Applicability, FatalError}; +use rustc_errors::FatalError; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_middle::ty::subst::{GenericArg, InternalSubsts, Subst}; @@ -21,11 +22,12 @@ use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitor, WithConstnes use rustc_middle::ty::{Predicate, ToPredicate}; use rustc_session::lint::builtin::WHERE_CLAUSES_OBJECT_SAFETY; use rustc_span::symbol::Symbol; -use rustc_span::Span; +use rustc_span::{MultiSpan, Span}; use smallvec::SmallVec; use std::array; use std::iter; +use std::ops::ControlFlow; pub use crate::traits::{MethodViolationCode, ObjectSafetyViolation}; @@ -100,49 +102,7 @@ fn object_safety_violations_for_trait( span, ) = violation { - // Using `CRATE_NODE_ID` is wrong, but it's hard to get a more precise id. - // It's also hard to get a use site span, so we use the method definition span. - tcx.struct_span_lint_hir( - WHERE_CLAUSES_OBJECT_SAFETY, - hir::CRATE_HIR_ID, - *span, - |lint| { - let mut err = lint.build(&format!( - "the trait `{}` cannot be made into an object", - tcx.def_path_str(trait_def_id) - )); - let node = tcx.hir().get_if_local(trait_def_id); - let msg = if let Some(hir::Node::Item(item)) = node { - err.span_label( - item.ident.span, - "this trait cannot be made into an object...", - ); - format!("...because {}", violation.error_msg()) - } else { - format!( - "the trait cannot be made into an object because {}", - violation.error_msg() - ) - }; - err.span_label(*span, &msg); - match (node, violation.solution()) { - (Some(_), Some((note, None))) => { - err.help(¬e); - } - (Some(_), Some((note, Some((sugg, span))))) => { - err.span_suggestion( - span, - ¬e, - sugg, - Applicability::MachineApplicable, - ); - } - // Only provide the help if its a local trait, otherwise it's not actionable. - _ => {} - } - err.emit(); - }, - ); + lint_object_unsafe_trait(tcx, *span, trait_def_id, violation); false } else { true @@ -180,6 +140,51 @@ fn object_safety_violations_for_trait( violations } +/// Lint object-unsafe trait. +fn lint_object_unsafe_trait( + tcx: TyCtxt<'_>, + span: Span, + trait_def_id: DefId, + violation: &ObjectSafetyViolation, +) { + // Using `CRATE_NODE_ID` is wrong, but it's hard to get a more precise id. + // It's also hard to get a use site span, so we use the method definition span. + tcx.struct_span_lint_hir(WHERE_CLAUSES_OBJECT_SAFETY, hir::CRATE_HIR_ID, span, |lint| { + let mut err = lint.build(&format!( + "the trait `{}` cannot be made into an object", + tcx.def_path_str(trait_def_id) + )); + let node = tcx.hir().get_if_local(trait_def_id); + let mut spans = MultiSpan::from_span(span); + if let Some(hir::Node::Item(item)) = node { + spans.push_span_label( + item.ident.span, + "this trait cannot be made into an object...".into(), + ); + spans.push_span_label(span, format!("...because {}", violation.error_msg())); + } else { + spans.push_span_label( + span, + format!( + "the trait cannot be made into an object because {}", + violation.error_msg() + ), + ); + }; + err.span_note( + spans, + "for a trait to be \"object safe\" it needs to allow building a vtable to allow the \ + call to be resolvable dynamically; for more information visit \ + <https://doc.rust-lang.org/reference/items/traits.html#object-safety>", + ); + if node.is_some() { + // Only provide the help if its a local trait, otherwise it's not + violation.solution(&mut err); + } + err.emit(); + }); +} + fn sized_trait_bound_spans<'tcx>( tcx: TyCtxt<'tcx>, bounds: hir::GenericBounds<'tcx>, @@ -246,7 +251,7 @@ fn predicates_reference_self( predicates .predicates .iter() - .map(|(predicate, sp)| (predicate.subst_supertrait(tcx, &trait_ref), *sp)) + .map(|&(predicate, sp)| (predicate.subst_supertrait(tcx, &trait_ref), sp)) .filter_map(|predicate| predicate_references_self(tcx, predicate)) .collect() } @@ -257,7 +262,7 @@ fn bounds_reference_self(tcx: TyCtxt<'_>, trait_def_id: DefId) -> SmallVec<[Span .in_definition_order() .filter(|item| item.kind == ty::AssocKind::Type) .flat_map(|item| tcx.explicit_item_bounds(item.def_id)) - .map(|(predicate, sp)| (predicate.subst_supertrait(tcx, &trait_ref), *sp)) + .map(|&(predicate, sp)| (predicate.subst_supertrait(tcx, &trait_ref), sp)) .filter_map(|predicate| predicate_references_self(tcx, predicate)) .collect() } @@ -385,6 +390,8 @@ fn virtual_call_violation_for_method<'tcx>( trait_def_id: DefId, method: &ty::AssocItem, ) -> Option<MethodViolationCode> { + let sig = tcx.fn_sig(method.def_id); + // The method's first parameter must be named `self` if !method.fn_has_self_parameter { // We'll attempt to provide a structured suggestion for `Self: Sized`. @@ -395,12 +402,22 @@ fn virtual_call_violation_for_method<'tcx>( [.., pred] => (", Self: Sized", pred.span().shrink_to_hi()), }, ); - return Some(MethodViolationCode::StaticMethod(sugg)); + // Get the span pointing at where the `self` receiver should be. + let sm = tcx.sess.source_map(); + let self_span = method.ident.span.to(tcx + .hir() + .span_if_local(method.def_id) + .unwrap_or_else(|| sm.next_point(method.ident.span)) + .shrink_to_hi()); + let self_span = sm.span_through_char(self_span, '(').shrink_to_hi(); + return Some(MethodViolationCode::StaticMethod( + sugg, + self_span, + !sig.inputs().skip_binder().is_empty(), + )); } - let sig = tcx.fn_sig(method.def_id); - - for (i, input_ty) in sig.skip_binder().inputs()[1..].iter().enumerate() { + for (i, &input_ty) in sig.skip_binder().inputs()[1..].iter().enumerate() { if contains_illegal_self_type_reference(tcx, trait_def_id, input_ty) { return Some(MethodViolationCode::ReferencesSelfInput(i)); } @@ -423,10 +440,7 @@ fn virtual_call_violation_for_method<'tcx>( // so outlives predicates will always hold. .cloned() .filter(|(p, _)| p.to_opt_type_outlives().is_none()) - .collect::<Vec<_>>() - // Do a shallow visit so that `contains_illegal_self_type_reference` - // may apply it's custom visiting. - .visit_tys_shallow(|t| contains_illegal_self_type_reference(tcx, trait_def_id, t)) + .any(|pred| contains_illegal_self_type_reference(tcx, trait_def_id, pred)) { return Some(MethodViolationCode::WhereClauseReferencesSelf); } @@ -448,10 +462,17 @@ fn virtual_call_violation_for_method<'tcx>( let param_env = tcx.param_env(method.def_id); - let abi_of_ty = |ty: Ty<'tcx>| -> &Abi { + let abi_of_ty = |ty: Ty<'tcx>| -> Option<&Abi> { match tcx.layout_of(param_env.and(ty)) { - Ok(layout) => &layout.abi, - Err(err) => bug!("error: {}\n while computing layout for type {:?}", err, ty), + Ok(layout) => Some(&layout.abi), + Err(err) => { + // #78372 + tcx.sess.delay_span_bug( + tcx.def_span(method.def_id), + &format!("error: {}\n while computing layout for type {:?}", err, ty), + ); + None + } } }; @@ -460,7 +481,7 @@ fn virtual_call_violation_for_method<'tcx>( receiver_for_self_ty(tcx, receiver_ty, tcx.mk_unit(), method.def_id); match abi_of_ty(unit_receiver_ty) { - &Abi::Scalar(..) => (), + Some(Abi::Scalar(..)) => (), abi => { tcx.sess.delay_span_bug( tcx.def_span(method.def_id), @@ -480,13 +501,12 @@ fn virtual_call_violation_for_method<'tcx>( receiver_for_self_ty(tcx, receiver_ty, trait_object_ty, method.def_id); match abi_of_ty(trait_object_receiver) { - &Abi::ScalarPair(..) => (), + Some(Abi::ScalarPair(..)) => (), abi => { tcx.sess.delay_span_bug( tcx.def_span(method.def_id), &format!( - "receiver when `Self = {}` should have a ScalarPair ABI; \ - found {:?}", + "receiver when `Self = {}` should have a ScalarPair ABI; found {:?}", trait_object_ty, abi ), ); @@ -700,10 +720,10 @@ fn receiver_is_dispatchable<'tcx>( }) } -fn contains_illegal_self_type_reference<'tcx>( +fn contains_illegal_self_type_reference<'tcx, T: TypeFoldable<'tcx>>( tcx: TyCtxt<'tcx>, trait_def_id: DefId, - ty: Ty<'tcx>, + value: T, ) -> bool { // This is somewhat subtle. In general, we want to forbid // references to `Self` in the argument and return types, @@ -746,15 +766,20 @@ fn contains_illegal_self_type_reference<'tcx>( struct IllegalSelfTypeVisitor<'tcx> { tcx: TyCtxt<'tcx>, - self_ty: Ty<'tcx>, trait_def_id: DefId, supertraits: Option<Vec<ty::PolyTraitRef<'tcx>>>, } impl<'tcx> TypeVisitor<'tcx> for IllegalSelfTypeVisitor<'tcx> { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { match t.kind() { - ty::Param(_) => t == self.self_ty, + ty::Param(_) => { + if t == self.tcx.types.self_param { + ControlFlow::BREAK + } else { + ControlFlow::CONTINUE + } + } ty::Projection(ref data) => { // This is a projected type `<Foo as SomeTrait>::X`. @@ -778,7 +803,7 @@ fn contains_illegal_self_type_reference<'tcx>( self.supertraits.as_ref().unwrap().contains(&projection_trait_ref); if is_supertrait_of_current_trait { - false // do not walk contained types, do not report error, do collect $200 + ControlFlow::CONTINUE // do not walk contained types, do not report error, do collect $200 } else { t.super_visit_with(self) // DO walk contained types, POSSIBLY reporting an error } @@ -787,22 +812,66 @@ fn contains_illegal_self_type_reference<'tcx>( } } - fn visit_const(&mut self, _c: &ty::Const<'tcx>) -> bool { - // FIXME(#72219) Look into the unevaluated constants for object safety violations. - // Do not walk substitutions of unevaluated consts, as they contain `Self`, even - // though the const expression doesn't necessary use it. Currently type variables - // inside array length expressions are forbidden, so they can't break the above - // rules. - false + fn visit_const(&mut self, ct: &ty::Const<'tcx>) -> ControlFlow<()> { + // First check if the type of this constant references `Self`. + self.visit_ty(ct.ty)?; + + // Constants can only influence object safety if they reference `Self`. + // This is only possible for unevaluated constants, so we walk these here. + // + // If `AbstractConst::new` returned an error we already failed compilation + // so we don't have to emit an additional error here. + // + // We currently recurse into abstract consts here but do not recurse in + // `is_const_evaluatable`. This means that the object safety check is more + // liberal than the const eval check. + // + // This shouldn't really matter though as we can't really use any + // constants which are not considered const evaluatable. + use rustc_middle::mir::abstract_const::Node; + if let Ok(Some(ct)) = AbstractConst::from_const(self.tcx, ct) { + const_evaluatable::walk_abstract_const(self.tcx, ct, |node| match node { + Node::Leaf(leaf) => { + let leaf = leaf.subst(self.tcx, ct.substs); + self.visit_const(leaf) + } + Node::Binop(..) | Node::UnaryOp(..) | Node::FunctionCall(_, _) => { + ControlFlow::CONTINUE + } + }) + } else { + ControlFlow::CONTINUE + } + } + + fn visit_predicate(&mut self, pred: ty::Predicate<'tcx>) -> ControlFlow<()> { + if let ty::PredicateAtom::ConstEvaluatable(def, substs) = pred.skip_binders() { + // FIXME(const_evaluatable_checked): We should probably deduplicate the logic for + // `AbstractConst`s here, it might make sense to change `ConstEvaluatable` to + // take a `ty::Const` instead. + use rustc_middle::mir::abstract_const::Node; + if let Ok(Some(ct)) = AbstractConst::new(self.tcx, def, substs) { + const_evaluatable::walk_abstract_const(self.tcx, ct, |node| match node { + Node::Leaf(leaf) => { + let leaf = leaf.subst(self.tcx, ct.substs); + self.visit_const(leaf) + } + Node::Binop(..) | Node::UnaryOp(..) | Node::FunctionCall(_, _) => { + ControlFlow::CONTINUE + } + }) + } else { + ControlFlow::CONTINUE + } + } else { + pred.super_visit_with(self) + } } } - ty.visit_with(&mut IllegalSelfTypeVisitor { - tcx, - self_ty: tcx.types.self_param, - trait_def_id, - supertraits: None, - }) + value + .visit_with(&mut IllegalSelfTypeVisitor { tcx, trait_def_id, supertraits: None }) + .is_break() } pub fn provide(providers: &mut ty::query::Providers) { diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs index fdf1641c986..b0bfb4ad173 100644 --- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs +++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs @@ -642,24 +642,30 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { debug!(?poly_trait_ref, "assemble_candidates_from_object_ty"); + let poly_trait_predicate = self.infcx().resolve_vars_if_possible(&obligation.predicate); + let placeholder_trait_predicate = + self.infcx().replace_bound_vars_with_placeholders(&poly_trait_predicate); + // Count only those upcast versions that match the trait-ref // we are looking for. Specifically, do not only check for the // correct trait, but also the correct type parameters. // For example, we may be trying to upcast `Foo` to `Bar<i32>`, // but `Foo` is declared as `trait Foo: Bar<u32>`. - let upcast_trait_refs = util::supertraits(self.tcx(), poly_trait_ref) - .filter(|upcast_trait_ref| { - self.infcx - .probe(|_| self.match_poly_trait_ref(obligation, *upcast_trait_ref).is_ok()) + let candidate_supertraits = util::supertraits(self.tcx(), poly_trait_ref) + .enumerate() + .filter(|&(_, upcast_trait_ref)| { + self.infcx.probe(|_| { + self.match_normalize_trait_ref( + obligation, + upcast_trait_ref, + placeholder_trait_predicate.trait_ref, + ) + .is_ok() + }) }) - .count(); + .map(|(idx, _)| ObjectCandidate(idx)); - if upcast_trait_refs > 1 { - // Can be upcast in many ways; need more type information. - candidates.ambiguous = true; - } else if upcast_trait_refs == 1 { - candidates.vec.push(ObjectCandidate); - } + candidates.vec.extend(candidate_supertraits); }) } diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index 37d619d5942..872b8e85f56 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -69,10 +69,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } ProjectionCandidate(idx) => { - let obligations = self.confirm_projection_candidate(obligation, idx); + let obligations = self.confirm_projection_candidate(obligation, idx)?; Ok(ImplSource::Param(obligations)) } + ObjectCandidate(idx) => { + let data = self.confirm_object_candidate(obligation, idx)?; + Ok(ImplSource::Object(data)) + } + ClosureCandidate => { let vtable_closure = self.confirm_closure_candidate(obligation)?; Ok(ImplSource::Closure(vtable_closure)) @@ -97,11 +102,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { Ok(ImplSource::TraitAlias(data)) } - ObjectCandidate => { - let data = self.confirm_object_candidate(obligation); - Ok(ImplSource::Object(data)) - } - BuiltinObjectCandidate => { // This indicates something like `Trait + Send: Send`. In this case, we know that // this holds because that's what the object type is telling us, and there's really @@ -120,7 +120,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { &mut self, obligation: &TraitObligation<'tcx>, idx: usize, - ) -> Vec<PredicateObligation<'tcx>> { + ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> { self.infcx.commit_unconditionally(|_| { let tcx = self.tcx(); @@ -148,19 +148,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { &mut obligations, ); - obligations.extend( + obligations.extend(self.infcx.commit_if_ok(|_| { self.infcx .at(&obligation.cause, obligation.param_env) .sup(placeholder_trait_predicate.trait_ref.to_poly_trait_ref(), candidate) .map(|InferOk { obligations, .. }| obligations) - .unwrap_or_else(|_| { - bug!( - "Projection bound `{:?}` was applicable to `{:?}` but now is not", - candidate, - obligation - ); - }), - ); + .map_err(|_| Unimplemented) + })?); if let ty::Projection(..) = placeholder_self_ty.kind() { for predicate in tcx.predicates_of(def_id).instantiate_own(tcx, substs).predicates { @@ -181,7 +175,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } } - obligations + Ok(obligations) }) } @@ -371,9 +365,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { fn confirm_object_candidate( &mut self, obligation: &TraitObligation<'tcx>, - ) -> ImplSourceObjectData<'tcx, PredicateObligation<'tcx>> { - debug!(?obligation, "confirm_object_candidate"); + index: usize, + ) -> Result<ImplSourceObjectData<'tcx, PredicateObligation<'tcx>>, SelectionError<'tcx>> { let tcx = self.tcx(); + debug!(?obligation, ?index, "confirm_object_candidate"); let trait_predicate = self.infcx.replace_bound_vars_with_placeholders(&obligation.predicate); @@ -399,43 +394,39 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { }) .with_self_ty(self.tcx(), self_ty); - let mut upcast_trait_ref = None; let mut nested = vec![]; - let vtable_base; - { - // We want to find the first supertrait in the list of - // supertraits that we can unify with, and do that - // unification. We know that there is exactly one in the list - // where we can unify, because otherwise select would have - // reported an ambiguity. (When we do find a match, also - // record it for later.) - let nonmatching = util::supertraits(tcx, ty::Binder::dummy(object_trait_ref)) - .take_while(|&t| { - match self.infcx.commit_if_ok(|_| { - self.infcx - .at(&obligation.cause, obligation.param_env) - .sup(obligation_trait_ref, t) - .map(|InferOk { obligations, .. }| obligations) - .map_err(|_| ()) - }) { - Ok(obligations) => { - upcast_trait_ref = Some(t); - nested.extend(obligations); - false - } - Err(_) => true, - } - }); + let mut supertraits = util::supertraits(tcx, ty::Binder::dummy(object_trait_ref)); - // Additionally, for each of the non-matching predicates that - // we pass over, we sum up the set of number of vtable - // entries, so that we can compute the offset for the selected - // trait. - vtable_base = nonmatching.map(|t| super::util::count_own_vtable_entries(tcx, t)).sum(); - } + // For each of the non-matching predicates that + // we pass over, we sum up the set of number of vtable + // entries, so that we can compute the offset for the selected + // trait. + let vtable_base = supertraits + .by_ref() + .take(index) + .map(|t| super::util::count_own_vtable_entries(tcx, t)) + .sum(); + + let unnormalized_upcast_trait_ref = + supertraits.next().expect("supertraits iterator no longer has as many elements"); + + let upcast_trait_ref = normalize_with_depth_to( + self, + obligation.param_env, + obligation.cause.clone(), + obligation.recursion_depth + 1, + &unnormalized_upcast_trait_ref, + &mut nested, + ); - let upcast_trait_ref = upcast_trait_ref.unwrap(); + nested.extend(self.infcx.commit_if_ok(|_| { + self.infcx + .at(&obligation.cause, obligation.param_env) + .sup(obligation_trait_ref, upcast_trait_ref) + .map(|InferOk { obligations, .. }| obligations) + .map_err(|_| Unimplemented) + })?); // Check supertraits hold. This is so that their associated type bounds // will be checked in the code below. @@ -501,7 +492,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { } debug!(?nested, "object nested obligations"); - ImplSourceObjectData { upcast_trait_ref, vtable_base, nested } + Ok(ImplSourceObjectData { upcast_trait_ref, vtable_base, nested }) } fn confirm_fn_pointer_candidate( diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs index b838602e76c..4cc4bc0acda 100644 --- a/compiler/rustc_trait_selection/src/traits/select/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs @@ -518,12 +518,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { result } Ok(Ok(None)) => Ok(EvaluatedToAmbig), - // EvaluatedToRecur might also be acceptable here, but use - // Unknown for now because it means that we won't dismiss a - // selection candidate solely because it has a projection - // cycle. This is closest to the previous behavior of - // immediately erroring. - Ok(Err(project::InProgress)) => Ok(EvaluatedToUnknown), + Ok(Err(project::InProgress)) => Ok(EvaluatedToRecur), Err(_) => Ok(EvaluatedToErr), } } @@ -1179,7 +1174,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { if let ty::PredicateAtom::Trait(pred, _) = bound_predicate.skip_binder() { let bound = bound_predicate.rebind(pred.trait_ref); if self.infcx.probe(|_| { - match self.match_projection( + match self.match_normalize_trait_ref( obligation, bound, placeholder_trait_predicate.trait_ref, @@ -1207,7 +1202,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { /// Equates the trait in `obligation` with trait bound. If the two traits /// can be equated and the normalized trait bound doesn't contain inference /// variables or placeholders, the normalized bound is returned. - fn match_projection( + fn match_normalize_trait_ref( &mut self, obligation: &TraitObligation<'tcx>, trait_bound: ty::PolyTraitRef<'tcx>, @@ -1357,10 +1352,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | BuiltinUnsizeCandidate | BuiltinCandidate { .. } | TraitAliasCandidate(..) - | ObjectCandidate + | ObjectCandidate(_) | ProjectionCandidate(_), ) => !is_global(cand), - (ObjectCandidate | ProjectionCandidate(_), ParamCandidate(ref cand)) => { + (ObjectCandidate(_) | ProjectionCandidate(_), ParamCandidate(ref cand)) => { // Prefer these to a global where-clause bound // (see issue #50825). is_global(cand) @@ -1381,20 +1376,20 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { is_global(cand) && other.evaluation.must_apply_modulo_regions() } - (ProjectionCandidate(i), ProjectionCandidate(j)) => { - // Arbitrarily pick the first candidate for backwards + (ProjectionCandidate(i), ProjectionCandidate(j)) + | (ObjectCandidate(i), ObjectCandidate(j)) => { + // Arbitrarily pick the lower numbered candidate for backwards // compatibility reasons. Don't let this affect inference. - i > j && !needs_infer + i < j && !needs_infer } - (ObjectCandidate, ObjectCandidate) => bug!("Duplicate object candidate"), - (ObjectCandidate, ProjectionCandidate(_)) - | (ProjectionCandidate(_), ObjectCandidate) => { + (ObjectCandidate(_), ProjectionCandidate(_)) + | (ProjectionCandidate(_), ObjectCandidate(_)) => { bug!("Have both object and projection candidate") } // Arbitrarily give projection and object candidates priority. ( - ObjectCandidate | ProjectionCandidate(_), + ObjectCandidate(_) | ProjectionCandidate(_), ImplCandidate(..) | ClosureCandidate | GeneratorCandidate @@ -1414,7 +1409,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { | BuiltinUnsizeCandidate | BuiltinCandidate { .. } | TraitAliasCandidate(..), - ObjectCandidate | ProjectionCandidate(_), + ObjectCandidate(_) | ProjectionCandidate(_), ) => false, (&ImplCandidate(other_def), &ImplCandidate(victim_def)) => { @@ -1890,9 +1885,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { /// Normalize `where_clause_trait_ref` and try to match it against /// `obligation`. If successful, return any predicates that - /// result from the normalization. Normalization is necessary - /// because where-clauses are stored in the parameter environment - /// unnormalized. + /// result from the normalization. fn match_where_clause_trait_ref( &mut self, obligation: &TraitObligation<'tcx>, diff --git a/compiler/rustc_trait_selection/src/traits/structural_match.rs b/compiler/rustc_trait_selection/src/traits/structural_match.rs index 4f7fa2c3988..ce0d3ef8a6a 100644 --- a/compiler/rustc_trait_selection/src/traits/structural_match.rs +++ b/compiler/rustc_trait_selection/src/traits/structural_match.rs @@ -8,6 +8,7 @@ use rustc_hir::lang_items::LangItem; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt, TypeFoldable, TypeVisitor}; use rustc_span::Span; +use std::ops::ControlFlow; #[derive(Debug)] pub enum NonStructuralMatchTy<'tcx> { @@ -134,38 +135,38 @@ impl Search<'a, 'tcx> { } impl<'a, 'tcx> TypeVisitor<'tcx> for Search<'a, 'tcx> { - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { debug!("Search visiting ty: {:?}", ty); let (adt_def, substs) = match *ty.kind() { ty::Adt(adt_def, substs) => (adt_def, substs), ty::Param(_) => { self.found = Some(NonStructuralMatchTy::Param); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Dynamic(..) => { self.found = Some(NonStructuralMatchTy::Dynamic); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Foreign(_) => { self.found = Some(NonStructuralMatchTy::Foreign); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Opaque(..) => { self.found = Some(NonStructuralMatchTy::Opaque); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Projection(..) => { self.found = Some(NonStructuralMatchTy::Projection); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Generator(..) | ty::GeneratorWitness(..) => { self.found = Some(NonStructuralMatchTy::Generator); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::Closure(..) => { self.found = Some(NonStructuralMatchTy::Closure); - return true; // Stop visiting. + return ControlFlow::BREAK; } ty::RawPtr(..) => { // structural-match ignores substructure of @@ -182,39 +183,31 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for Search<'a, 'tcx> { // Even though `NonStructural` does not implement `PartialEq`, // structural equality on `T` does not recur into the raw // pointer. Therefore, one can still use `C` in a pattern. - - // (But still tell the caller to continue search.) - return false; + return ControlFlow::CONTINUE; } ty::FnDef(..) | ty::FnPtr(..) => { // Types of formals and return in `fn(_) -> _` are also irrelevant; // so we do not recur into them via `super_visit_with` - // - // (But still tell the caller to continue search.) - return false; + return ControlFlow::CONTINUE; } ty::Array(_, n) if { n.try_eval_usize(self.tcx(), ty::ParamEnv::reveal_all()) == Some(0) } => { // rust-lang/rust#62336: ignore type of contents // for empty array. - // - // (But still tell the caller to continue search.) - return false; + return ControlFlow::CONTINUE; } ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Str | ty::Never => { // These primitive types are always structural match. // // `Never` is kind of special here, but as it is not inhabitable, this should be fine. - // - // (But still tell the caller to continue search.) - return false; + return ControlFlow::CONTINUE; } ty::Array(..) | ty::Slice(_) | ty::Ref(..) | ty::Tuple(..) => { // First check all contained types and then tell the caller to continue searching. ty.super_visit_with(self); - return false; + return ControlFlow::CONTINUE; } ty::Infer(_) | ty::Placeholder(_) | ty::Bound(..) => { bug!("unexpected type during structural-match checking: {:?}", ty); @@ -223,22 +216,19 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for Search<'a, 'tcx> { self.tcx().sess.delay_span_bug(self.span, "ty::Error in structural-match check"); // We still want to check other types after encountering an error, // as this may still emit relevant errors. - // - // So we continue searching here. - return false; + return ControlFlow::CONTINUE; } }; if !self.seen.insert(adt_def.did) { debug!("Search already seen adt_def: {:?}", adt_def); - // Let caller continue its search. - return false; + return ControlFlow::CONTINUE; } if !self.type_marked_structural(ty) { debug!("Search found ty: {:?}", ty); self.found = Some(NonStructuralMatchTy::Adt(&adt_def)); - return true; // Halt visiting! + return ControlFlow::BREAK; } // structural-match does not care about the @@ -258,16 +248,16 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for Search<'a, 'tcx> { let ty = self.tcx().normalize_erasing_regions(ty::ParamEnv::empty(), field_ty); debug!("structural-match ADT: field_ty={:?}, ty={:?}", field_ty, ty); - if ty.visit_with(self) { + if ty.visit_with(self).is_break() { // found an ADT without structural-match; halt visiting! assert!(self.found.is_some()); - return true; + return ControlFlow::BREAK; } } // Even though we do not want to recur on substs, we do // want our caller to continue its own search. - false + ControlFlow::CONTINUE } } diff --git a/compiler/rustc_traits/src/chalk/db.rs b/compiler/rustc_traits/src/chalk/db.rs index 3368c5b7699..e5ae899a2f3 100644 --- a/compiler/rustc_traits/src/chalk/db.rs +++ b/compiler/rustc_traits/src/chalk/db.rs @@ -342,29 +342,29 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t } (ty::Bool, Scalar(Bool)) => true, (ty::Char, Scalar(Char)) => true, - (ty::Int(ty1), Scalar(Int(ty2))) => match (ty1, ty2) { + (ty::Int(ty1), Scalar(Int(ty2))) => matches!( + (ty1, ty2), (ast::IntTy::Isize, chalk_ir::IntTy::Isize) - | (ast::IntTy::I8, chalk_ir::IntTy::I8) - | (ast::IntTy::I16, chalk_ir::IntTy::I16) - | (ast::IntTy::I32, chalk_ir::IntTy::I32) - | (ast::IntTy::I64, chalk_ir::IntTy::I64) - | (ast::IntTy::I128, chalk_ir::IntTy::I128) => true, - _ => false, - }, - (ty::Uint(ty1), Scalar(Uint(ty2))) => match (ty1, ty2) { + | (ast::IntTy::I8, chalk_ir::IntTy::I8) + | (ast::IntTy::I16, chalk_ir::IntTy::I16) + | (ast::IntTy::I32, chalk_ir::IntTy::I32) + | (ast::IntTy::I64, chalk_ir::IntTy::I64) + | (ast::IntTy::I128, chalk_ir::IntTy::I128) + ), + (ty::Uint(ty1), Scalar(Uint(ty2))) => matches!( + (ty1, ty2), (ast::UintTy::Usize, chalk_ir::UintTy::Usize) - | (ast::UintTy::U8, chalk_ir::UintTy::U8) - | (ast::UintTy::U16, chalk_ir::UintTy::U16) - | (ast::UintTy::U32, chalk_ir::UintTy::U32) - | (ast::UintTy::U64, chalk_ir::UintTy::U64) - | (ast::UintTy::U128, chalk_ir::UintTy::U128) => true, - _ => false, - }, - (ty::Float(ty1), Scalar(Float(ty2))) => match (ty1, ty2) { + | (ast::UintTy::U8, chalk_ir::UintTy::U8) + | (ast::UintTy::U16, chalk_ir::UintTy::U16) + | (ast::UintTy::U32, chalk_ir::UintTy::U32) + | (ast::UintTy::U64, chalk_ir::UintTy::U64) + | (ast::UintTy::U128, chalk_ir::UintTy::U128) + ), + (ty::Float(ty1), Scalar(Float(ty2))) => matches!( + (ty1, ty2), (ast::FloatTy::F32, chalk_ir::FloatTy::F32) - | (ast::FloatTy::F64, chalk_ir::FloatTy::F64) => true, - _ => false, - }, + | (ast::FloatTy::F64, chalk_ir::FloatTy::F64) + ), (&ty::Tuple(..), Tuple(..)) => true, (&ty::Array(..), Array) => true, (&ty::Slice(..), Slice) => true, diff --git a/compiler/rustc_traits/src/chalk/lowering.rs b/compiler/rustc_traits/src/chalk/lowering.rs index 5ca0fc0c88b..01c4dd12487 100644 --- a/compiler/rustc_traits/src/chalk/lowering.rs +++ b/compiler/rustc_traits/src/chalk/lowering.rs @@ -42,6 +42,7 @@ use rustc_span::def_id::DefId; use chalk_ir::{FnSig, ForeignDefId}; use rustc_hir::Unsafety; use std::collections::btree_map::{BTreeMap, Entry}; +use std::ops::ControlFlow; /// Essentially an `Into` with a `&RustInterner` parameter crate trait LowerInto<'tcx, T> { @@ -897,14 +898,14 @@ impl<'tcx> BoundVarsCollector<'tcx> { } impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> { - fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> bool { + fn visit_binder<T: TypeFoldable<'tcx>>(&mut self, t: &Binder<T>) -> ControlFlow<()> { self.binder_index.shift_in(1); let result = t.super_visit_with(self); self.binder_index.shift_out(1); result } - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { match *t.kind() { ty::Bound(debruijn, bound_ty) if debruijn == self.binder_index => { match self.parameters.entry(bound_ty.var.as_u32()) { @@ -924,7 +925,7 @@ impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> { t.super_visit_with(self) } - fn visit_region(&mut self, r: Region<'tcx>) -> bool { + fn visit_region(&mut self, r: Region<'tcx>) -> ControlFlow<()> { match r { ty::ReLateBound(index, br) if *index == self.binder_index => match br { ty::BoundRegion::BrNamed(def_id, _name) => { @@ -1114,7 +1115,7 @@ impl PlaceholdersCollector { } impl<'tcx> TypeVisitor<'tcx> for PlaceholdersCollector { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { match t.kind() { ty::Placeholder(p) if p.universe == self.universe_index => { self.next_ty_placeholder = self.next_ty_placeholder.max(p.name.as_usize() + 1); @@ -1126,7 +1127,7 @@ impl<'tcx> TypeVisitor<'tcx> for PlaceholdersCollector { t.super_visit_with(self) } - fn visit_region(&mut self, r: Region<'tcx>) -> bool { + fn visit_region(&mut self, r: Region<'tcx>) -> ControlFlow<()> { match r { ty::RePlaceholder(p) if p.universe == self.universe_index => { if let ty::BoundRegion::BrAnon(anon) = p.name { diff --git a/compiler/rustc_traits/src/implied_outlives_bounds.rs b/compiler/rustc_traits/src/implied_outlives_bounds.rs index bc5c07fce04..c44fd1d5859 100644 --- a/compiler/rustc_traits/src/implied_outlives_bounds.rs +++ b/compiler/rustc_traits/src/implied_outlives_bounds.rs @@ -62,7 +62,7 @@ fn compute_implied_outlives_bounds<'tcx>( // unresolved inference variables here anyway, but there might be // during typeck under some circumstances.) let obligations = wf::obligations(infcx, param_env, hir::CRATE_HIR_ID, 0, arg, DUMMY_SP) - .unwrap_or(vec![]); + .unwrap_or_default(); // N.B., all of these predicates *ought* to be easily proven // true. In fact, their correctness is (mostly) implied by diff --git a/compiler/rustc_traits/src/lib.rs b/compiler/rustc_traits/src/lib.rs index d0b05beb4e6..7b688cd3e21 100644 --- a/compiler/rustc_traits/src/lib.rs +++ b/compiler/rustc_traits/src/lib.rs @@ -4,6 +4,7 @@ #![feature(crate_visibility_modifier)] #![feature(in_band_lifetimes)] #![feature(nll)] +#![feature(control_flow_enum)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_ty/src/ty.rs b/compiler/rustc_ty/src/ty.rs index 1f21d9488a4..2562140bb5d 100644 --- a/compiler/rustc_ty/src/ty.rs +++ b/compiler/rustc_ty/src/ty.rs @@ -80,7 +80,6 @@ fn sized_constraint_for_ty<'tcx>( fn associated_item_from_trait_item_ref( tcx: TyCtxt<'_>, parent_def_id: LocalDefId, - parent_vis: &hir::Visibility<'_>, trait_item_ref: &hir::TraitItemRef, ) -> ty::AssocItem { let def_id = tcx.hir().local_def_id(trait_item_ref.id.hir_id); @@ -93,8 +92,7 @@ fn associated_item_from_trait_item_ref( ty::AssocItem { ident: trait_item_ref.ident, kind, - // Visibility of trait items is inherited from their traits. - vis: ty::Visibility::from_hir(parent_vis, trait_item_ref.id.hir_id, tcx), + vis: tcx.visibility(def_id), defaultness: trait_item_ref.defaultness, def_id: def_id.to_def_id(), container: ty::TraitContainer(parent_def_id.to_def_id()), @@ -117,8 +115,7 @@ fn associated_item_from_impl_item_ref( ty::AssocItem { ident: impl_item_ref.ident, kind, - // Visibility of trait impl items doesn't matter. - vis: ty::Visibility::from_hir(&impl_item_ref.vis, impl_item_ref.id.hir_id, tcx), + vis: tcx.visibility(def_id), defaultness: impl_item_ref.defaultness, def_id: def_id.to_def_id(), container: ty::ImplContainer(parent_def_id.to_def_id()), @@ -143,12 +140,8 @@ fn associated_item(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItem { hir::ItemKind::Trait(.., ref trait_item_refs) => { if let Some(trait_item_ref) = trait_item_refs.iter().find(|i| i.id.hir_id == id) { - let assoc_item = associated_item_from_trait_item_ref( - tcx, - parent_def_id, - &parent_item.vis, - trait_item_ref, - ); + let assoc_item = + associated_item_from_trait_item_ref(tcx, parent_def_id, trait_item_ref); debug_assert_eq!(assoc_item.def_id, def_id); return assoc_item; } diff --git a/compiler/rustc_typeck/src/astconv/generics.rs b/compiler/rustc_typeck/src/astconv/generics.rs index b867798c76c..3bfb2d3f1b0 100644 --- a/compiler/rustc_typeck/src/astconv/generics.rs +++ b/compiler/rustc_typeck/src/astconv/generics.rs @@ -548,13 +548,18 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o { generics: &ty::Generics, ) -> bool { let explicit = !seg.infer_args; - let impl_trait = generics.params.iter().any(|param| match param.kind { - ty::GenericParamDefKind::Type { - synthetic: Some(hir::SyntheticTyParamKind::ImplTrait), - .. - } => true, - _ => false, - }); + let impl_trait = + generics.params.iter().any(|param| match param.kind { + ty::GenericParamDefKind::Type { + synthetic: + Some( + hir::SyntheticTyParamKind::ImplTrait + | hir::SyntheticTyParamKind::FromAttr, + ), + .. + } => true, + _ => false, + }); if explicit && impl_trait { let spans = seg diff --git a/compiler/rustc_typeck/src/check/_match.rs b/compiler/rustc_typeck/src/check/_match.rs index 7cb23dc0537..e8eea65137f 100644 --- a/compiler/rustc_typeck/src/check/_match.rs +++ b/compiler/rustc_typeck/src/check/_match.rs @@ -9,6 +9,7 @@ use rustc_trait_selection::opaque_types::InferCtxtExt as _; use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use rustc_trait_selection::traits::{ IfExpressionCause, MatchExpressionArmCause, ObligationCause, ObligationCauseCode, + StatementAsExpression, }; impl<'a, 'tcx> FnCtxt<'a, 'tcx> { @@ -188,11 +189,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } } else { - let (arm_span, semi_span) = if let hir::ExprKind::Block(blk, _) = &arm.body.kind { - self.find_block_span(blk, prior_arm_ty) - } else { - (arm.body.span, None) - }; + let (arm_span, semi_span) = + self.get_appropriate_arm_semicolon_removal_span(&arms, i, prior_arm_ty, arm_ty); let (span, code) = match i { // The reason for the first arm to fail is not that the match arms diverge, // but rather that there's a prior obligation that doesn't hold. @@ -201,6 +199,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { expr.span, ObligationCauseCode::MatchExpressionArm(box MatchExpressionArmCause { arm_span, + scrut_span: scrut.span, semi_span, source: match_src, prior_arms: other_arms.clone(), @@ -241,6 +240,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { coercion.complete(self) } + fn get_appropriate_arm_semicolon_removal_span( + &self, + arms: &'tcx [hir::Arm<'tcx>], + i: usize, + prior_arm_ty: Option<Ty<'tcx>>, + arm_ty: Ty<'tcx>, + ) -> (Span, Option<(Span, StatementAsExpression)>) { + let arm = &arms[i]; + let (arm_span, mut semi_span) = if let hir::ExprKind::Block(blk, _) = &arm.body.kind { + self.find_block_span(blk, prior_arm_ty) + } else { + (arm.body.span, None) + }; + if semi_span.is_none() && i > 0 { + if let hir::ExprKind::Block(blk, _) = &arms[i - 1].body.kind { + let (_, semi_span_prev) = self.find_block_span(blk, Some(arm_ty)); + semi_span = semi_span_prev; + } + } + (arm_span, semi_span) + } + /// When the previously checked expression (the scrutinee) diverges, /// warn the user about the match arms being unreachable. fn warn_arms_when_scrutinee_diverges( @@ -513,7 +534,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { &self, block: &'tcx hir::Block<'tcx>, expected_ty: Option<Ty<'tcx>>, - ) -> (Span, Option<Span>) { + ) -> (Span, Option<(Span, StatementAsExpression)>) { if let Some(expr) = &block.expr { (expr.span, None) } else if let Some(stmt) = block.stmts.last() { diff --git a/compiler/rustc_typeck/src/check/check.rs b/compiler/rustc_typeck/src/check/check.rs index 3d8653b4a6a..70d94ef869d 100644 --- a/compiler/rustc_typeck/src/check/check.rs +++ b/compiler/rustc_typeck/src/check/check.rs @@ -14,8 +14,9 @@ use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt}; use rustc_middle::ty::fold::TypeFoldable; use rustc_middle::ty::subst::GenericArgKind; use rustc_middle::ty::util::{Discr, IntTypeExt, Representability}; -use rustc_middle::ty::{self, RegionKind, ToPredicate, Ty, TyCtxt}; +use rustc_middle::ty::{self, ParamEnv, RegionKind, ToPredicate, Ty, TyCtxt}; use rustc_session::config::EntryFnType; +use rustc_session::lint::builtin::UNINHABITED_STATIC; use rustc_span::symbol::sym; use rustc_span::{self, MultiSpan, Span}; use rustc_target::spec::abi::Abi; @@ -23,6 +24,8 @@ use rustc_trait_selection::opaque_types::InferCtxtExt as _; use rustc_trait_selection::traits::error_reporting::InferCtxtExt as _; use rustc_trait_selection::traits::{self, ObligationCauseCode}; +use std::ops::ControlFlow; + pub fn check_wf_new(tcx: TyCtxt<'_>) { let visit = wfcheck::CheckTypeWellFormedVisitor::new(tcx); tcx.hir().krate().par_visit_all_item_likes(&visit); @@ -130,7 +133,7 @@ pub(super) fn check_fn<'a, 'tcx>( // The check for a non-trivial pattern is a hack to avoid duplicate warnings // for simple cases like `fn foo(x: Trait)`, // where we would error once on the parameter as a whole, and once on the binding `x`. - if param.pat.simple_ident().is_none() && !tcx.features().unsized_locals { + if param.pat.simple_ident().is_none() && !tcx.features().unsized_fn_params { fcx.require_type_is_sized(param_ty, param.pat.span, traits::SizedArgumentType(ty_span)); } @@ -338,7 +341,7 @@ pub(super) fn check_struct(tcx: TyCtxt<'_>, id: hir::HirId, span: Span) { check_packed(tcx, span, def); } -pub(super) fn check_union(tcx: TyCtxt<'_>, id: hir::HirId, span: Span) { +fn check_union(tcx: TyCtxt<'_>, id: hir::HirId, span: Span) { let def_id = tcx.hir().local_def_id(id); let def = tcx.adt_def(def_id); def.destructor(tcx); // force the destructor to be evaluated @@ -349,7 +352,7 @@ pub(super) fn check_union(tcx: TyCtxt<'_>, id: hir::HirId, span: Span) { } /// Check that the fields of the `union` do not need dropping. -pub(super) fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> bool { +fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> bool { let item_type = tcx.type_of(item_def_id); if let ty::Adt(def, substs) = item_type.kind() { assert!(def.is_union()); @@ -377,6 +380,36 @@ pub(super) fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: Local true } +/// Check that a `static` is inhabited. +fn check_static_inhabited<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId, span: Span) { + // Make sure statics are inhabited. + // Other parts of the compiler assume that there are no uninhabited places. In principle it + // would be enough to check this for `extern` statics, as statics with an initializer will + // have UB during initialization if they are uninhabited, but there also seems to be no good + // reason to allow any statics to be uninhabited. + let ty = tcx.type_of(def_id); + let layout = match tcx.layout_of(ParamEnv::reveal_all().and(ty)) { + Ok(l) => l, + Err(_) => { + // Generic statics are rejected, but we still reach this case. + tcx.sess.delay_span_bug(span, "generic static must be rejected"); + return; + } + }; + if layout.abi.is_uninhabited() { + tcx.struct_span_lint_hir( + UNINHABITED_STATIC, + tcx.hir().local_def_id_to_hir_id(def_id), + span, + |lint| { + lint.build("static of uninhabited type") + .note("uninhabited statics cannot be initialized, and any access would be an immediate error") + .emit(); + }, + ); + } +} + /// Checks that an opaque type does not contain cycles and does not use `Self` or `T::Foo` /// projections that would result in "inheriting lifetimes". pub(super) fn check_opaque<'tcx>( @@ -417,30 +450,34 @@ pub(super) fn check_opaque_for_inheriting_lifetimes( }; impl<'tcx> ty::fold::TypeVisitor<'tcx> for ProhibitOpaqueVisitor<'tcx> { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { debug!("check_opaque_for_inheriting_lifetimes: (visit_ty) t={:?}", t); - if t != self.opaque_identity_ty && t.super_visit_with(self) { + if t != self.opaque_identity_ty && t.super_visit_with(self).is_break() { self.ty = Some(t); - return true; + return ControlFlow::BREAK; } - false + ControlFlow::CONTINUE } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { debug!("check_opaque_for_inheriting_lifetimes: (visit_region) r={:?}", r); if let RegionKind::ReEarlyBound(ty::EarlyBoundRegion { index, .. }) = r { - return *index < self.generics.parent_count as u32; + if *index < self.generics.parent_count as u32 { + return ControlFlow::BREAK; + } else { + return ControlFlow::CONTINUE; + } } r.super_visit_with(self) } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { if let ty::ConstKind::Unevaluated(..) = c.val { // FIXME(#72219) We currenctly don't detect lifetimes within substs // which would violate this check. Even though the particular substitution is not used // within the const, this should still be fixed. - return false; + return ControlFlow::CONTINUE; } c.super_visit_with(self) } @@ -462,7 +499,7 @@ pub(super) fn check_opaque_for_inheriting_lifetimes( let prohibit_opaque = tcx .explicit_item_bounds(def_id) .iter() - .any(|(predicate, _)| predicate.visit_with(&mut visitor)); + .any(|(predicate, _)| predicate.visit_with(&mut visitor).is_break()); debug!( "check_opaque_for_inheriting_lifetimes: prohibit_opaque={:?}, visitor={:?}", prohibit_opaque, visitor @@ -609,6 +646,7 @@ pub fn check_item_type<'tcx>(tcx: TyCtxt<'tcx>, it: &'tcx hir::Item<'tcx>) { let def_id = tcx.hir().local_def_id(it.hir_id); tcx.ensure().typeck(def_id); maybe_check_static_with_link_section(tcx, def_id, it.span); + check_static_inhabited(tcx, def_id, it.span); } hir::ItemKind::Const(..) => { tcx.ensure().typeck(tcx.hir().local_def_id(it.hir_id)); @@ -691,7 +729,8 @@ pub fn check_item_type<'tcx>(tcx: TyCtxt<'tcx>, it: &'tcx hir::Item<'tcx>) { } } else { for item in m.items { - let generics = tcx.generics_of(tcx.hir().local_def_id(item.hir_id)); + let def_id = tcx.hir().local_def_id(item.hir_id); + let generics = tcx.generics_of(def_id); let own_counts = generics.own_counts(); if generics.params.len() - own_counts.lifetimes != 0 { let (kinds, kinds_pl, egs) = match (own_counts.types, own_counts.consts) { @@ -722,8 +761,14 @@ pub fn check_item_type<'tcx>(tcx: TyCtxt<'tcx>, it: &'tcx hir::Item<'tcx>) { .emit(); } - if let hir::ForeignItemKind::Fn(ref fn_decl, _, _) = item.kind { - require_c_abi_if_c_variadic(tcx, fn_decl, m.abi, item.span); + match item.kind { + hir::ForeignItemKind::Fn(ref fn_decl, _, _) => { + require_c_abi_if_c_variadic(tcx, fn_decl, m.abi, item.span); + } + hir::ForeignItemKind::Static(..) => { + check_static_inhabited(tcx, def_id, item.span); + } + _ => {} } } } @@ -1410,11 +1455,11 @@ fn opaque_type_cycle_error(tcx: TyCtxt<'tcx>, def_id: LocalDefId, span: Span) { { struct VisitTypes(Vec<DefId>); impl<'tcx> ty::fold::TypeVisitor<'tcx> for VisitTypes { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { match *t.kind() { ty::Opaque(def, _) => { self.0.push(def); - false + ControlFlow::CONTINUE } _ => t.super_visit_with(self), } diff --git a/compiler/rustc_typeck/src/check/coercion.rs b/compiler/rustc_typeck/src/check/coercion.rs index c1485e3baf6..6da3ecde329 100644 --- a/compiler/rustc_typeck/src/check/coercion.rs +++ b/compiler/rustc_typeck/src/check/coercion.rs @@ -1475,6 +1475,28 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { if let (Some(sp), Some(fn_output)) = (fcx.ret_coercion_span.borrow().as_ref(), fn_output) { self.add_impl_trait_explanation(&mut err, cause, fcx, expected, *sp, fn_output); } + + if let Some(sp) = fcx.ret_coercion_span.borrow().as_ref() { + // If the closure has an explicit return type annotation, + // then a type error may occur at the first return expression we + // see in the closure (if it conflicts with the declared + // return type). Skip adding a note in this case, since it + // would be incorrect. + if !err.span.primary_spans().iter().any(|span| span == sp) { + let hir = fcx.tcx.hir(); + let body_owner = hir.body_owned_by(hir.enclosing_body_owner(fcx.body_id)); + if fcx.tcx.is_closure(hir.body_owner_def_id(body_owner).to_def_id()) { + err.span_note( + *sp, + &format!( + "return type inferred to be `{}` here", + fcx.resolve_vars_if_possible(&expected) + ), + ); + } + } + } + err } diff --git a/compiler/rustc_typeck/src/check/demand.rs b/compiler/rustc_typeck/src/check/demand.rs index 3e66885448b..241803fab1e 100644 --- a/compiler/rustc_typeck/src/check/demand.rs +++ b/compiler/rustc_typeck/src/check/demand.rs @@ -33,7 +33,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { return; } self.suggest_boxing_when_appropriate(err, expr, expected, expr_ty); - self.suggest_missing_await(err, expr, expected, expr_ty); self.suggest_missing_parentheses(err, expr); self.note_need_for_fn_pointer(err, expected, expr_ty); self.note_internal_mutation_in_method(err, expr, expected, expr_ty); @@ -751,8 +750,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } - let msg = format!("you can convert an `{}` to `{}`", checked_ty, expected_ty); - let cast_msg = format!("you can cast an `{} to `{}`", checked_ty, expected_ty); + let msg = format!( + "you can convert {} `{}` to {} `{}`", + checked_ty.kind().article(), + checked_ty, + expected_ty.kind().article(), + expected_ty, + ); + let cast_msg = format!( + "you can cast {} `{}` to {} `{}`", + checked_ty.kind().article(), + checked_ty, + expected_ty.kind().article(), + expected_ty, + ); let lit_msg = format!( "change the type of the numeric literal from `{}` to `{}`", checked_ty, expected_ty, @@ -814,7 +825,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let suggestion = format!("{}::from({})", checked_ty, lhs_src); (lhs_expr.span, msg, suggestion) } else { - let msg = format!("{} and panic if the converted value wouldn't fit", msg); + let msg = format!("{} and panic if the converted value doesn't fit", msg); let suggestion = format!("{}{}.try_into().unwrap()", prefix, with_opt_paren(&src)); (expr.span, msg, suggestion) diff --git a/compiler/rustc_typeck/src/check/expr.rs b/compiler/rustc_typeck/src/check/expr.rs index 9990f86a36b..324aa1a66a6 100644 --- a/compiler/rustc_typeck/src/check/expr.rs +++ b/compiler/rustc_typeck/src/check/expr.rs @@ -42,7 +42,7 @@ use rustc_middle::ty::{AdtKind, Visibility}; use rustc_span::hygiene::DesugaringKind; use rustc_span::source_map::Span; use rustc_span::symbol::{kw, sym, Ident, Symbol}; -use rustc_trait_selection::traits::{self, ObligationCauseCode, SelectionContext}; +use rustc_trait_selection::traits::{self, ObligationCauseCode}; use std::fmt::Display; @@ -476,7 +476,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { if let ty::FnDef(..) = ty.kind() { let fn_sig = ty.fn_sig(tcx); - if !tcx.features().unsized_locals { + if !tcx.features().unsized_fn_params { // We want to remove some Sized bounds from std functions, // but don't want to expose the removal to stable Rust. // i.e., we don't want to allow @@ -627,7 +627,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { assert!(expr_opt.is_none() || self.tcx.sess.has_errors()); } - ctxt.may_break = true; + // If we encountered a `break`, then (no surprise) it may be possible to break from the + // loop... unless the value being returned from the loop diverges itself, e.g. + // `break return 5` or `break loop {}`. + ctxt.may_break |= !self.diverges.get().is_always(); // the type of a `break` is always `!`, since it diverges tcx.types.never @@ -769,34 +772,40 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let mut err = self.demand_suptype_diag(expr.span, expected_ty, actual_ty).unwrap(); let lhs_ty = self.check_expr(&lhs); let rhs_ty = self.check_expr(&rhs); - if self.can_coerce(lhs_ty, rhs_ty) { - if !lhs.is_syntactic_place_expr() { - // Do not suggest `if let x = y` as `==` is way more likely to be the intention. - if let hir::Node::Expr(hir::Expr { - kind: ExprKind::Match(_, _, hir::MatchSource::IfDesugar { .. }), - .. - }) = self.tcx.hir().get( - self.tcx.hir().get_parent_node(self.tcx.hir().get_parent_node(expr.hir_id)), - ) { - // Likely `if let` intended. - err.span_suggestion_verbose( - expr.span.shrink_to_lo(), - "you might have meant to use pattern matching", - "let ".to_string(), - Applicability::MaybeIncorrect, - ); - } + let (applicability, eq) = if self.can_coerce(rhs_ty, lhs_ty) { + (Applicability::MachineApplicable, true) + } else { + (Applicability::MaybeIncorrect, false) + }; + if !lhs.is_syntactic_place_expr() { + // Do not suggest `if let x = y` as `==` is way more likely to be the intention. + if let hir::Node::Expr(hir::Expr { + kind: + ExprKind::Match( + _, + _, + hir::MatchSource::IfDesugar { .. } | hir::MatchSource::WhileDesugar, + ), + .. + }) = self.tcx.hir().get( + self.tcx.hir().get_parent_node(self.tcx.hir().get_parent_node(expr.hir_id)), + ) { + // Likely `if let` intended. + err.span_suggestion_verbose( + expr.span.shrink_to_lo(), + "you might have meant to use pattern matching", + "let ".to_string(), + applicability, + ); } + } + if eq { err.span_suggestion_verbose( *span, "you might have meant to compare for equality", "==".to_string(), - Applicability::MaybeIncorrect, + applicability, ); - } else { - // Do this to cause extra errors about the assignment. - let lhs_ty = self.check_expr_with_needs(&lhs, Needs::MutPlace); - let _ = self.check_expr_coercable_to_type(&rhs, lhs_ty, Some(lhs)); } if self.sess().if_let_suggestions.borrow().get(&expr.span).is_some() { @@ -1574,51 +1583,34 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { err: &mut DiagnosticBuilder<'_>, field_ident: Ident, base: &'tcx hir::Expr<'tcx>, - expr: &'tcx hir::Expr<'tcx>, - def_id: DefId, + ty: Ty<'tcx>, ) { - let param_env = self.tcx().param_env(def_id); - let future_trait = self.tcx.require_lang_item(LangItem::Future, None); - // Future::Output - let item_def_id = - self.tcx.associated_items(future_trait).in_definition_order().next().unwrap().def_id; - - let projection_ty = self.tcx.projection_ty_from_predicates((def_id, item_def_id)); - debug!("suggest_await_on_field_access: projection_ty={:?}", projection_ty); - - let cause = self.misc(expr.span); - let mut selcx = SelectionContext::new(&self.infcx); - - let mut obligations = vec![]; - if let Some(projection_ty) = projection_ty { - let normalized_ty = rustc_trait_selection::traits::normalize_projection_type( - &mut selcx, - param_env, - projection_ty, - cause, - 0, - &mut obligations, - ); - debug!( - "suggest_await_on_field_access: normalized_ty={:?}, ty_kind={:?}", - self.resolve_vars_if_possible(&normalized_ty), - normalized_ty.kind(), - ); - if let ty::Adt(def, _) = normalized_ty.kind() { - // no field access on enum type - if !def.is_enum() { - if def.non_enum_variant().fields.iter().any(|field| field.ident == field_ident) - { - err.span_suggestion_verbose( - base.span.shrink_to_hi(), - "consider awaiting before field access", - ".await".to_string(), - Applicability::MaybeIncorrect, - ); - } + let output_ty = match self.infcx.get_impl_future_output_ty(ty) { + Some(output_ty) => self.resolve_vars_if_possible(&output_ty), + _ => return, + }; + let mut add_label = true; + if let ty::Adt(def, _) = output_ty.kind() { + // no field access on enum type + if !def.is_enum() { + if def.non_enum_variant().fields.iter().any(|field| field.ident == field_ident) { + add_label = false; + err.span_label( + field_ident.span, + "field not available in `impl Future`, but it is available in its `Output`", + ); + err.span_suggestion_verbose( + base.span.shrink_to_hi(), + "consider `await`ing on the `Future` and access the field of its `Output`", + ".await".to_string(), + Applicability::MaybeIncorrect, + ); } } } + if add_label { + err.span_label(field_ident.span, &format!("field not found in `{}`", ty)); + } } fn ban_nonexisting_field( @@ -1647,8 +1639,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ty::Param(param_ty) => { self.point_at_param_definition(&mut err, param_ty); } - ty::Opaque(def_id, _) => { - self.suggest_await_on_field_access(&mut err, field, base, expr, def_id); + ty::Opaque(_, _) => { + self.suggest_await_on_field_access(&mut err, field, base, expr_t.peel_refs()); } _ => {} } diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs index 017b0abd1d6..f87e6b607d4 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/_impl.rs @@ -33,7 +33,9 @@ use rustc_span::{self, BytePos, MultiSpan, Span}; use rustc_trait_selection::infer::InferCtxtExt as _; use rustc_trait_selection::opaque_types::InferCtxtExt as _; use rustc_trait_selection::traits::error_reporting::InferCtxtExt as _; -use rustc_trait_selection::traits::{self, ObligationCauseCode, TraitEngine, TraitEngineExt}; +use rustc_trait_selection::traits::{ + self, ObligationCauseCode, StatementAsExpression, TraitEngine, TraitEngineExt, +}; use std::collections::hash_map::Entry; use std::slice; @@ -1061,7 +1063,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { &self, blk: &'tcx hir::Block<'tcx>, expected_ty: Ty<'tcx>, - ) -> Option<Span> { + ) -> Option<(Span, StatementAsExpression)> { // Be helpful when the user wrote `{... expr;}` and // taking the `;` off is enough to fix the error. let last_stmt = blk.stmts.last()?; @@ -1070,13 +1072,58 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { _ => return None, }; let last_expr_ty = self.node_ty(last_expr.hir_id); - if matches!(last_expr_ty.kind(), ty::Error(_)) - || self.can_sub(self.param_env, last_expr_ty, expected_ty).is_err() + let needs_box = match (last_expr_ty.kind(), expected_ty.kind()) { + (ty::Opaque(last_def_id, last_bounds), ty::Opaque(exp_def_id, exp_bounds)) => { + debug!( + "both opaque, likely future {:?} {:?} {:?} {:?}", + last_def_id, last_bounds, exp_def_id, exp_bounds + ); + let last_hir_id = self.tcx.hir().local_def_id_to_hir_id(last_def_id.expect_local()); + let exp_hir_id = self.tcx.hir().local_def_id_to_hir_id(exp_def_id.expect_local()); + match ( + &self.tcx.hir().expect_item(last_hir_id).kind, + &self.tcx.hir().expect_item(exp_hir_id).kind, + ) { + ( + hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: last_bounds, .. }), + hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: exp_bounds, .. }), + ) if last_bounds.iter().zip(exp_bounds.iter()).all(|(left, right)| { + match (left, right) { + ( + hir::GenericBound::Trait(tl, ml), + hir::GenericBound::Trait(tr, mr), + ) if tl.trait_ref.trait_def_id() == tr.trait_ref.trait_def_id() + && ml == mr => + { + true + } + ( + hir::GenericBound::LangItemTrait(langl, _, _, argsl), + hir::GenericBound::LangItemTrait(langr, _, _, argsr), + ) if langl == langr => { + // FIXME: consider the bounds! + debug!("{:?} {:?}", argsl, argsr); + true + } + _ => false, + } + }) => + { + StatementAsExpression::NeedsBoxing + } + _ => StatementAsExpression::CorrectType, + } + } + _ => StatementAsExpression::CorrectType, + }; + if (matches!(last_expr_ty.kind(), ty::Error(_)) + || self.can_sub(self.param_env, last_expr_ty, expected_ty).is_err()) + && matches!(needs_box, StatementAsExpression::CorrectType) { return None; } let original_span = original_sp(last_stmt.span, blk.span); - Some(original_span.with_lo(original_span.hi() - BytePos(1))) + Some((original_span.with_lo(original_span.hi() - BytePos(1)), needs_box)) } // Instantiates the given path, which must refer to an item with the given diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs index fd2f5eb5018..a820661d843 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/checks.rs @@ -20,7 +20,7 @@ use rustc_middle::ty::{self, Ty}; use rustc_session::Session; use rustc_span::symbol::{sym, Ident}; use rustc_span::{self, MultiSpan, Span}; -use rustc_trait_selection::traits::{self, ObligationCauseCode}; +use rustc_trait_selection::traits::{self, ObligationCauseCode, StatementAsExpression}; use std::mem::replace; use std::slice; @@ -758,13 +758,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { expected_ty: Ty<'tcx>, err: &mut DiagnosticBuilder<'_>, ) { - if let Some(span_semi) = self.could_remove_semicolon(blk, expected_ty) { - err.span_suggestion( - span_semi, - "consider removing this semicolon", - String::new(), - Applicability::MachineApplicable, - ); + if let Some((span_semi, boxed)) = self.could_remove_semicolon(blk, expected_ty) { + if let StatementAsExpression::NeedsBoxing = boxed { + err.span_suggestion_verbose( + span_semi, + "consider removing this semicolon and boxing the expression", + String::new(), + Applicability::HasPlaceholders, + ); + } else { + err.span_suggestion_short( + span_semi, + "consider removing this semicolon", + String::new(), + Applicability::MachineApplicable, + ); + } } } diff --git a/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs b/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs index 9bad02c41b4..a8ad9f4fdf8 100644 --- a/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs +++ b/compiler/rustc_typeck/src/check/fn_ctxt/suggestions.rs @@ -3,7 +3,6 @@ use crate::astconv::AstConv; use rustc_ast::util::parser::ExprPrecedence; use rustc_span::{self, Span}; -use rustc_trait_selection::traits; use rustc_errors::{Applicability, DiagnosticBuilder}; use rustc_hir as hir; @@ -13,7 +12,6 @@ use rustc_hir::{ExprKind, ItemKind, Node}; use rustc_infer::infer; use rustc_middle::ty::{self, Ty}; use rustc_span::symbol::kw; -use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _; use std::iter; @@ -433,87 +431,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } } - /// A possible error is to forget to add `.await` when using futures: - /// - /// ``` - /// async fn make_u32() -> u32 { - /// 22 - /// } - /// - /// fn take_u32(x: u32) {} - /// - /// async fn foo() { - /// let x = make_u32(); - /// take_u32(x); - /// } - /// ``` - /// - /// This routine checks if the found type `T` implements `Future<Output=U>` where `U` is the - /// expected type. If this is the case, and we are inside of an async body, it suggests adding - /// `.await` to the tail of the expression. - pub(in super::super) fn suggest_missing_await( - &self, - err: &mut DiagnosticBuilder<'_>, - expr: &hir::Expr<'_>, - expected: Ty<'tcx>, - found: Ty<'tcx>, - ) { - debug!("suggest_missing_await: expr={:?} expected={:?}, found={:?}", expr, expected, found); - // `.await` is not permitted outside of `async` bodies, so don't bother to suggest if the - // body isn't `async`. - let item_id = self.tcx().hir().get_parent_node(self.body_id); - if let Some(body_id) = self.tcx().hir().maybe_body_owned_by(item_id) { - let body = self.tcx().hir().body(body_id); - if let Some(hir::GeneratorKind::Async(_)) = body.generator_kind { - let sp = expr.span; - // Check for `Future` implementations by constructing a predicate to - // prove: `<T as Future>::Output == U` - let future_trait = self.tcx.require_lang_item(LangItem::Future, Some(sp)); - let item_def_id = self - .tcx - .associated_items(future_trait) - .in_definition_order() - .next() - .unwrap() - .def_id; - // `<T as Future>::Output` - let projection_ty = ty::ProjectionTy { - // `T` - substs: self - .tcx - .mk_substs_trait(found, self.fresh_substs_for_item(sp, item_def_id)), - // `Future::Output` - item_def_id, - }; - - let predicate = ty::PredicateAtom::Projection(ty::ProjectionPredicate { - projection_ty, - ty: expected, - }) - .potentially_quantified(self.tcx, ty::PredicateKind::ForAll); - let obligation = traits::Obligation::new(self.misc(sp), self.param_env, predicate); - - debug!("suggest_missing_await: trying obligation {:?}", obligation); - - if self.infcx.predicate_may_hold(&obligation) { - debug!("suggest_missing_await: obligation held: {:?}", obligation); - if let Ok(code) = self.sess().source_map().span_to_snippet(sp) { - err.span_suggestion( - sp, - "consider using `.await` here", - format!("{}.await", code), - Applicability::MaybeIncorrect, - ); - } else { - debug!("suggest_missing_await: no snippet for {:?}", sp); - } - } else { - debug!("suggest_missing_await: obligation did not hold: {:?}", obligation) - } - } - } - } - pub(in super::super) fn suggest_missing_parentheses( &self, err: &mut DiagnosticBuilder<'_>, diff --git a/compiler/rustc_typeck/src/check/gather_locals.rs b/compiler/rustc_typeck/src/check/gather_locals.rs index 1d505cfa698..af552389de0 100644 --- a/compiler/rustc_typeck/src/check/gather_locals.rs +++ b/compiler/rustc_typeck/src/check/gather_locals.rs @@ -6,15 +6,20 @@ use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKi use rustc_middle::ty::Ty; use rustc_span::Span; use rustc_trait_selection::traits; +use std::mem; pub(super) struct GatherLocalsVisitor<'a, 'tcx> { fcx: &'a FnCtxt<'a, 'tcx>, parent_id: hir::HirId, + // parameters are special cases of patterns, but we want to handle them as + // *distinct* cases. so track when we are hitting a pattern *within* an fn + // parameter. + outermost_fn_param_pat: bool, } impl<'a, 'tcx> GatherLocalsVisitor<'a, 'tcx> { pub(super) fn new(fcx: &'a FnCtxt<'a, 'tcx>, parent_id: hir::HirId) -> Self { - Self { fcx, parent_id } + Self { fcx, parent_id, outermost_fn_param_pat: false } } fn assign(&mut self, span: Span, nid: hir::HirId, ty_opt: Option<LocalTy<'tcx>>) -> Ty<'tcx> { @@ -88,13 +93,29 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> { intravisit::walk_local(self, local); } + fn visit_param(&mut self, param: &'tcx hir::Param<'tcx>) { + let old_outermost_fn_param_pat = mem::replace(&mut self.outermost_fn_param_pat, true); + intravisit::walk_param(self, param); + self.outermost_fn_param_pat = old_outermost_fn_param_pat; + } + // Add pattern bindings. fn visit_pat(&mut self, p: &'tcx hir::Pat<'tcx>) { if let PatKind::Binding(_, _, ident, _) = p.kind { let var_ty = self.assign(p.span, p.hir_id, None); - if !self.fcx.tcx.features().unsized_locals { - self.fcx.require_type_is_sized(var_ty, p.span, traits::VariableType(p.hir_id)); + if self.outermost_fn_param_pat { + if !self.fcx.tcx.features().unsized_fn_params { + self.fcx.require_type_is_sized( + var_ty, + p.span, + traits::SizedArgumentType(Some(p.span)), + ); + } + } else { + if !self.fcx.tcx.features().unsized_locals { + self.fcx.require_type_is_sized(var_ty, p.span, traits::VariableType(p.hir_id)); + } } debug!( @@ -104,7 +125,9 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> { var_ty ); } + let old_outermost_fn_param_pat = mem::replace(&mut self.outermost_fn_param_pat, false); intravisit::walk_pat(self, p); + self.outermost_fn_param_pat = old_outermost_fn_param_pat; } // Don't descend into the bodies of nested closures. diff --git a/compiler/rustc_typeck/src/check/generator_interior.rs b/compiler/rustc_typeck/src/check/generator_interior.rs index 3fc5f02a4a4..293a995887c 100644 --- a/compiler/rustc_typeck/src/check/generator_interior.rs +++ b/compiler/rustc_typeck/src/check/generator_interior.rs @@ -186,8 +186,9 @@ pub fn resolve_interior<'a, 'tcx>( // which means that none of the regions inside relate to any other, even if // typeck had previously found constraints that would cause them to be related. let folded = fcx.tcx.fold_regions(&erased, &mut false, |_, current_depth| { + let r = fcx.tcx.mk_region(ty::ReLateBound(current_depth, ty::BrAnon(counter))); counter += 1; - fcx.tcx.mk_region(ty::ReLateBound(current_depth, ty::BrAnon(counter))) + r }); cause.ty = folded; @@ -250,10 +251,7 @@ impl<'a, 'tcx> Visitor<'tcx> for InteriorVisitor<'a, 'tcx> { let mut scope_var_ids = self.guard_bindings.pop().expect("should have pushed at least one earlier"); for var_id in scope_var_ids.drain(..) { - assert!( - self.guard_bindings_set.remove(&var_id), - "variable should be placed in scope earlier" - ); + self.guard_bindings_set.remove(&var_id); } } self.visit_expr(body); @@ -347,6 +345,18 @@ impl<'a, 'tcx> Visitor<'tcx> for InteriorVisitor<'a, 'tcx> { // The type table might not have information for this expression // if it is in a malformed scope. (#66387) if let Some(ty) = self.fcx.typeck_results.borrow().expr_ty_opt(expr) { + if guard_borrowing_from_pattern { + // Match guards create references to all the bindings in the pattern that are used + // in the guard, e.g. `y if is_even(y) => ...` becomes `is_even(*r_y)` where `r_y` + // is a reference to `y`, so we must record a reference to the type of the binding. + let tcx = self.fcx.tcx; + let ref_ty = tcx.mk_ref( + // Use `ReErased` as `resolve_interior` is going to replace all the regions anyway. + tcx.mk_region(ty::RegionKind::ReErased), + ty::TypeAndMut { ty, mutbl: hir::Mutability::Not }, + ); + self.record(ref_ty, scope, Some(expr), expr.span, guard_borrowing_from_pattern); + } self.record(ty, scope, Some(expr), expr.span, guard_borrowing_from_pattern); } else { self.fcx.tcx.sess.delay_span_bug(expr.span, "no type for node"); diff --git a/compiler/rustc_typeck/src/check/method/suggest.rs b/compiler/rustc_typeck/src/check/method/suggest.rs index 6d2ffadc20c..46afe4892db 100644 --- a/compiler/rustc_typeck/src/check/method/suggest.rs +++ b/compiler/rustc_typeck/src/check/method/suggest.rs @@ -21,7 +21,6 @@ use rustc_span::symbol::{kw, sym, Ident}; use rustc_span::{source_map, FileName, Span}; use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use rustc_trait_selection::traits::Obligation; -use rustc_trait_selection::traits::SelectionContext; use std::cmp::Ordering; @@ -870,46 +869,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { call: &hir::Expr<'_>, span: Span, ) { - if let ty::Opaque(def_id, _) = *ty.kind() { - let future_trait = self.tcx.require_lang_item(LangItem::Future, None); - // Future::Output - let item_def_id = self - .tcx - .associated_items(future_trait) - .in_definition_order() - .next() - .unwrap() - .def_id; - - let projection_ty = self.tcx.projection_ty_from_predicates((def_id, item_def_id)); - let cause = self.misc(span); - let mut selcx = SelectionContext::new(&self.infcx); - let mut obligations = vec![]; - if let Some(projection_ty) = projection_ty { - let normalized_ty = rustc_trait_selection::traits::normalize_projection_type( - &mut selcx, - self.param_env, - projection_ty, - cause, - 0, - &mut obligations, - ); - debug!( - "suggest_await_before_method: normalized_ty={:?}, ty_kind={:?}", - self.resolve_vars_if_possible(&normalized_ty), - normalized_ty.kind(), - ); - let method_exists = self.method_exists(item_name, normalized_ty, call.hir_id, true); - debug!("suggest_await_before_method: is_method_exist={}", method_exists); - if method_exists { - err.span_suggestion_verbose( - span.shrink_to_lo(), - "consider awaiting before this method call", - "await.".to_string(), - Applicability::MaybeIncorrect, - ); - } - } + let output_ty = match self.infcx.get_impl_future_output_ty(ty) { + Some(output_ty) => self.resolve_vars_if_possible(&output_ty), + _ => return, + }; + let method_exists = self.method_exists(item_name, output_ty, call.hir_id, true); + debug!("suggest_await_before_method: is_method_exist={}", method_exists); + if method_exists { + err.span_suggestion_verbose( + span.shrink_to_lo(), + "consider `await`ing on the `Future` and calling the method on its `Output`", + "await.".to_string(), + Applicability::MaybeIncorrect, + ); } } diff --git a/compiler/rustc_typeck/src/check/op.rs b/compiler/rustc_typeck/src/check/op.rs index 2f3fb49717b..247b5256726 100644 --- a/compiler/rustc_typeck/src/check/op.rs +++ b/compiler/rustc_typeck/src/check/op.rs @@ -19,6 +19,8 @@ use rustc_span::symbol::{sym, Ident}; use rustc_span::Span; use rustc_trait_selection::infer::InferCtxtExt; +use std::ops::ControlFlow; + impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// Checks a `a <op>= b` pub fn check_binop_assign( @@ -302,7 +304,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { true, ), hir::BinOpKind::Mul => ( - format!("cannot multiply `{}` to `{}`", rhs_ty, lhs_ty), + format!("cannot multiply `{}` by `{}`", lhs_ty, rhs_ty), Some("std::ops::Mul"), true, ), @@ -981,7 +983,7 @@ fn suggest_constraining_param( struct TypeParamVisitor<'tcx>(Vec<Ty<'tcx>>); impl<'tcx> TypeVisitor<'tcx> for TypeParamVisitor<'tcx> { - fn visit_ty(&mut self, ty: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<()> { if let ty::Param(_) = ty.kind() { self.0.push(ty); } diff --git a/compiler/rustc_typeck/src/check/wfcheck.rs b/compiler/rustc_typeck/src/check/wfcheck.rs index b4e950ab6e9..1e27357ce44 100644 --- a/compiler/rustc_typeck/src/check/wfcheck.rs +++ b/compiler/rustc_typeck/src/check/wfcheck.rs @@ -24,6 +24,8 @@ use rustc_trait_selection::opaque_types::may_define_opaque_type; use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt; use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode}; +use std::ops::ControlFlow; + /// Helper type of a temporary returned by `.for_item(...)`. /// This is necessary because we can't write the following bound: /// @@ -798,18 +800,18 @@ fn check_where_clauses<'tcx, 'fcx>( params: FxHashSet<u32>, } impl<'tcx> ty::fold::TypeVisitor<'tcx> for CountParams { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { if let ty::Param(param) = t.kind() { self.params.insert(param.index); } t.super_visit_with(self) } - fn visit_region(&mut self, _: ty::Region<'tcx>) -> bool { - true + fn visit_region(&mut self, _: ty::Region<'tcx>) -> ControlFlow<()> { + ControlFlow::BREAK } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { if let ty::ConstKind::Param(param) = c.val { self.params.insert(param.index); } @@ -817,7 +819,7 @@ fn check_where_clauses<'tcx, 'fcx>( } } let mut param_count = CountParams::default(); - let has_region = pred.visit_with(&mut param_count); + let has_region = pred.visit_with(&mut param_count).is_break(); let substituted_pred = pred.subst(fcx.tcx, substs); // Don't check non-defaulted params, dependent defaults (including lifetimes) // or preds with multiple params. diff --git a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs index be77d049cae..ce157f809ef 100644 --- a/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs +++ b/compiler/rustc_typeck/src/coherence/inherent_impls_overlap.rs @@ -2,8 +2,9 @@ use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc_hir::itemlikevisit::ItemLikeVisitor; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; use rustc_trait_selection::traits::{self, SkipLeakCheck}; +use smallvec::SmallVec; pub fn crate_inherent_impls_overlap_check(tcx: TyCtxt<'_>, crate_num: CrateNum) { assert_eq!(crate_num, LOCAL_CRATE); @@ -18,9 +19,18 @@ struct InherentOverlapChecker<'tcx> { impl InherentOverlapChecker<'tcx> { /// Checks whether any associated items in impls 1 and 2 share the same identifier and /// namespace. - fn impls_have_common_items(&self, impl1: DefId, impl2: DefId) -> bool { - let impl_items1 = self.tcx.associated_items(impl1); - let impl_items2 = self.tcx.associated_items(impl2); + fn impls_have_common_items( + &self, + impl_items1: &ty::AssociatedItems<'_>, + impl_items2: &ty::AssociatedItems<'_>, + ) -> bool { + let mut impl_items1 = &impl_items1; + let mut impl_items2 = &impl_items2; + + // Performance optimization: iterate over the smaller list + if impl_items1.len() > impl_items2.len() { + std::mem::swap(&mut impl_items1, &mut impl_items2); + } for item1 in impl_items1.in_definition_order() { let collision = impl_items2.filter_by_name_unhygienic(item1.ident.name).any(|item2| { @@ -113,9 +123,20 @@ impl ItemLikeVisitor<'v> for InherentOverlapChecker<'tcx> { let ty_def_id = self.tcx.hir().local_def_id(item.hir_id); let impls = self.tcx.inherent_impls(ty_def_id); - for (i, &impl1_def_id) in impls.iter().enumerate() { - for &impl2_def_id in &impls[(i + 1)..] { - if self.impls_have_common_items(impl1_def_id, impl2_def_id) { + // If there is only one inherent impl block, + // there is nothing to overlap check it with + if impls.len() <= 1 { + return; + } + + let impls_items = impls + .iter() + .map(|impl_def_id| (impl_def_id, self.tcx.associated_items(*impl_def_id))) + .collect::<SmallVec<[_; 8]>>(); + + for (i, &(&impl1_def_id, impl_items1)) in impls_items.iter().enumerate() { + for &(&impl2_def_id, impl_items2) in &impls_items[(i + 1)..] { + if self.impls_have_common_items(impl_items1, impl_items2) { self.check_for_overlapping_inherent_impls(impl1_def_id, impl2_def_id); } } diff --git a/compiler/rustc_typeck/src/collect.rs b/compiler/rustc_typeck/src/collect.rs index 630e80d502e..f014ea3d5a9 100644 --- a/compiler/rustc_typeck/src/collect.rs +++ b/compiler/rustc_typeck/src/collect.rs @@ -40,7 +40,7 @@ use rustc_middle::ty::query::Providers; use rustc_middle::ty::subst::InternalSubsts; use rustc_middle::ty::util::Discr; use rustc_middle::ty::util::IntTypeExt; -use rustc_middle::ty::{self, AdtKind, Const, ToPolyTraitRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, AdtKind, Const, DefIdTree, ToPolyTraitRef, Ty, TyCtxt}; use rustc_middle::ty::{ReprOptions, ToPredicate, WithConstness}; use rustc_session::config::SanitizerSet; use rustc_session::lint; @@ -50,6 +50,8 @@ use rustc_span::{Span, DUMMY_SP}; use rustc_target::spec::abi; use rustc_trait_selection::traits::error_reporting::suggestions::NextTypeParamName; +use std::ops::ControlFlow; + mod item_bounds; mod type_of; @@ -851,7 +853,6 @@ fn convert_variant( parent_did: LocalDefId, ) -> ty::VariantDef { let mut seen_fields: FxHashMap<Ident, Span> = Default::default(); - let hir_id = tcx.hir().local_def_id_to_hir_id(variant_did.unwrap_or(parent_did)); let fields = def .fields() .iter() @@ -868,11 +869,7 @@ fn convert_variant( seen_fields.insert(f.ident.normalize_to_macros_2_0(), f.span); } - ty::FieldDef { - did: fid.to_def_id(), - ident: f.ident, - vis: ty::Visibility::from_hir(&f.vis, hir_id, tcx), - } + ty::FieldDef { did: fid.to_def_id(), ident: f.ident, vis: tcx.visibility(fid) } }) .collect(); let recovered = match def { @@ -2065,14 +2062,14 @@ fn const_evaluatable_predicates_of<'tcx>( } impl<'a, 'tcx> TypeVisitor<'tcx> for TyAliasVisitor<'a, 'tcx> { - fn visit_const(&mut self, ct: &'tcx Const<'tcx>) -> bool { + fn visit_const(&mut self, ct: &'tcx Const<'tcx>) -> ControlFlow<()> { if let ty::ConstKind::Unevaluated(def, substs, None) = ct.val { self.preds.insert(( ty::PredicateAtom::ConstEvaluatable(def, substs).to_predicate(self.tcx), self.span, )); } - false + ControlFlow::CONTINUE } } @@ -2095,25 +2092,25 @@ fn const_evaluatable_predicates_of<'tcx>( if let hir::Node::Item(item) = node { if let hir::ItemKind::Impl { ref of_trait, ref self_ty, .. } = item.kind { if let Some(of_trait) = of_trait { - warn!("const_evaluatable_predicates_of({:?}): visit impl trait_ref", def_id); + debug!("const_evaluatable_predicates_of({:?}): visit impl trait_ref", def_id); collector.visit_trait_ref(of_trait); } - warn!("const_evaluatable_predicates_of({:?}): visit_self_ty", def_id); + debug!("const_evaluatable_predicates_of({:?}): visit_self_ty", def_id); collector.visit_ty(self_ty); } } if let Some(generics) = node.generics() { - warn!("const_evaluatable_predicates_of({:?}): visit_generics", def_id); + debug!("const_evaluatable_predicates_of({:?}): visit_generics", def_id); collector.visit_generics(generics); } if let Some(fn_sig) = tcx.hir().fn_sig_by_hir_id(hir_id) { - warn!("const_evaluatable_predicates_of({:?}): visit_fn_decl", def_id); + debug!("const_evaluatable_predicates_of({:?}): visit_fn_decl", def_id); collector.visit_fn_decl(fn_sig.decl); } - warn!("const_evaluatable_predicates_of({:?}) = {:?}", def_id, collector.preds); + debug!("const_evaluatable_predicates_of({:?}) = {:?}", def_id, collector.preds); collector.preds } @@ -2414,6 +2411,7 @@ fn from_target_feature( Some(sym::movbe_target_feature) => rust_features.movbe_target_feature, Some(sym::rtm_target_feature) => rust_features.rtm_target_feature, Some(sym::f16c_target_feature) => rust_features.f16c_target_feature, + Some(sym::ermsb_target_feature) => rust_features.ermsb_target_feature, Some(name) => bug!("unknown target feature gate {}", name), None => true, }; @@ -2791,6 +2789,14 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, id: DefId) -> CodegenFnAttrs { } }); + // #73631: closures inherit `#[target_feature]` annotations + if tcx.features().target_feature_11 && tcx.is_closure(id) { + let owner_id = tcx.parent(id).expect("closure should have a parent"); + codegen_fn_attrs + .target_features + .extend(tcx.codegen_fn_attrs(owner_id).target_features.iter().copied()) + } + // If a function uses #[target_feature] it can't be inlined into general // purpose functions as they wouldn't have the right target features // enabled. For that reason we also forbid #[inline(always)] as it can't be diff --git a/compiler/rustc_typeck/src/collect/type_of.rs b/compiler/rustc_typeck/src/collect/type_of.rs index 5e8c6211408..61d1efc837b 100644 --- a/compiler/rustc_typeck/src/collect/type_of.rs +++ b/compiler/rustc_typeck/src/collect/type_of.rs @@ -79,7 +79,13 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option< let _tables = tcx.typeck(body_owner); &*path } - _ => span_bug!(DUMMY_SP, "unexpected const parent path {:?}", parent_node), + _ => { + tcx.sess.delay_span_bug( + tcx.def_span(def_id), + &format!("unexpected const parent path {:?}", parent_node), + ); + return None; + } }; // We've encountered an `AnonConst` in some path, so we need to @@ -112,12 +118,16 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option< tcx.sess.delay_span_bug(tcx.def_span(def_id), "anon const with Res::Err"); return None; } - _ => span_bug!( - DUMMY_SP, - "unexpected anon const res {:?} in path: {:?}", - res, - path, - ), + _ => { + // If the user tries to specify generics on a type that does not take them, + // e.g. `usize<T>`, we may hit this branch, in which case we treat it as if + // no arguments have been passed. An error should already have been emitted. + tcx.sess.delay_span_bug( + tcx.def_span(def_id), + &format!("unexpected anon const res {:?} in path: {:?}", res, path), + ); + return None; + } }; generics diff --git a/compiler/rustc_typeck/src/constrained_generic_params.rs b/compiler/rustc_typeck/src/constrained_generic_params.rs index 09b5a9b0a65..bae5bde7002 100644 --- a/compiler/rustc_typeck/src/constrained_generic_params.rs +++ b/compiler/rustc_typeck/src/constrained_generic_params.rs @@ -2,6 +2,7 @@ use rustc_data_structures::fx::FxHashSet; use rustc_middle::ty::fold::{TypeFoldable, TypeVisitor}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::source_map::Span; +use std::ops::ControlFlow; #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct Parameter(pub u32); @@ -56,11 +57,11 @@ struct ParameterCollector { } impl<'tcx> TypeVisitor<'tcx> for ParameterCollector { - fn visit_ty(&mut self, t: Ty<'tcx>) -> bool { + fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<()> { match *t.kind() { ty::Projection(..) | ty::Opaque(..) if !self.include_nonconstraining => { // projections are not injective - return false; + return ControlFlow::CONTINUE; } ty::Param(data) => { self.parameters.push(Parameter::from(data)); @@ -71,14 +72,14 @@ impl<'tcx> TypeVisitor<'tcx> for ParameterCollector { t.super_visit_with(self) } - fn visit_region(&mut self, r: ty::Region<'tcx>) -> bool { + fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<()> { if let ty::ReEarlyBound(data) = *r { self.parameters.push(Parameter::from(data)); } - false + ControlFlow::CONTINUE } - fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> bool { + fn visit_const(&mut self, c: &'tcx ty::Const<'tcx>) -> ControlFlow<()> { match c.val { ty::ConstKind::Unevaluated(..) if !self.include_nonconstraining => { // Constant expressions are not injective diff --git a/compiler/rustc_typeck/src/lib.rs b/compiler/rustc_typeck/src/lib.rs index c1fa39e96eb..30904091c1b 100644 --- a/compiler/rustc_typeck/src/lib.rs +++ b/compiler/rustc_typeck/src/lib.rs @@ -66,6 +66,7 @@ This API is completely unstable and subject to change. #![feature(try_blocks)] #![feature(never_type)] #![feature(slice_partition_dedup)] +#![feature(control_flow_enum)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_typeck/src/variance/mod.rs b/compiler/rustc_typeck/src/variance/mod.rs index a893f69c48a..1565efbb022 100644 --- a/compiler/rustc_typeck/src/variance/mod.rs +++ b/compiler/rustc_typeck/src/variance/mod.rs @@ -4,7 +4,7 @@ //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/variance.html use hir::Node; -use rustc_arena::TypedArena; +use rustc_arena::DroplessArena; use rustc_hir as hir; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc_middle::ty::query::Providers; @@ -32,8 +32,8 @@ pub fn provide(providers: &mut Providers) { fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateVariancesMap<'_> { assert_eq!(crate_num, LOCAL_CRATE); - let mut arena = TypedArena::default(); - let terms_cx = terms::determine_parameters_to_be_inferred(tcx, &mut arena); + let arena = DroplessArena::default(); + let terms_cx = terms::determine_parameters_to_be_inferred(tcx, &arena); let constraints_cx = constraints::add_constraints_from_crate(terms_cx); solve::solve_constraints(constraints_cx) } diff --git a/compiler/rustc_typeck/src/variance/terms.rs b/compiler/rustc_typeck/src/variance/terms.rs index f61a783de69..81c858c53cb 100644 --- a/compiler/rustc_typeck/src/variance/terms.rs +++ b/compiler/rustc_typeck/src/variance/terms.rs @@ -9,7 +9,7 @@ // `InferredIndex` is a newtype'd int representing the index of such // a variable. -use rustc_arena::TypedArena; +use rustc_arena::DroplessArena; use rustc_hir as hir; use rustc_hir::itemlikevisit::ItemLikeVisitor; use rustc_hir::HirIdMap; @@ -47,7 +47,7 @@ impl<'a> fmt::Debug for VarianceTerm<'a> { pub struct TermsContext<'a, 'tcx> { pub tcx: TyCtxt<'tcx>, - pub arena: &'a TypedArena<VarianceTerm<'a>>, + pub arena: &'a DroplessArena, // For marker types, UnsafeCell, and other lang items where // variance is hardcoded, records the item-id and the hardcoded @@ -64,7 +64,7 @@ pub struct TermsContext<'a, 'tcx> { pub fn determine_parameters_to_be_inferred<'a, 'tcx>( tcx: TyCtxt<'tcx>, - arena: &'a mut TypedArena<VarianceTerm<'a>>, + arena: &'a DroplessArena, ) -> TermsContext<'a, 'tcx> { let mut terms_cx = TermsContext { tcx, |