diff options
| author | Sayan Nandan <17377258+sntdevco@users.noreply.github.com> | 2019-08-09 13:01:05 +0530 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-08-09 13:01:05 +0530 |
| commit | fb3a01354ffecc41d7a189e4dd225d706387a522 (patch) | |
| tree | 41492dfe93f1dccba847dadb56ac6aa079edaaa9 /src/libsyntax | |
| parent | 33445aea509cadcd715009c79795d289268daa7c (diff) | |
| parent | 5aa3d9a7b5d3a46a7f158e8881146331a6bc9243 (diff) | |
| download | rust-fb3a01354ffecc41d7a189e4dd225d706387a522.tar.gz rust-fb3a01354ffecc41d7a189e4dd225d706387a522.zip | |
Merge pull request #1 from rust-lang/master
Merge recent changes into master
Diffstat (limited to 'src/libsyntax')
64 files changed, 13466 insertions, 13316 deletions
diff --git a/src/libsyntax/Cargo.toml b/src/libsyntax/Cargo.toml index 71c2ab82f65..d4a9acc1569 100644 --- a/src/libsyntax/Cargo.toml +++ b/src/libsyntax/Cargo.toml @@ -7,15 +7,18 @@ edition = "2018" [lib] name = "syntax" path = "lib.rs" -crate-type = ["dylib"] +doctest = false [dependencies] bitflags = "1.0" -serialize = { path = "../libserialize" } +rustc_serialize = { path = "../libserialize", package = "serialize" } log = "0.4" scoped-tls = "1.0" +lazy_static = "1.0.0" syntax_pos = { path = "../libsyntax_pos" } errors = { path = "../librustc_errors", package = "rustc_errors" } rustc_data_structures = { path = "../librustc_data_structures" } +rustc_lexer = { path = "../librustc_lexer" } +rustc_macros = { path = "../librustc_macros" } rustc_target = { path = "../librustc_target" } smallvec = { version = "0.6.7", features = ["union", "may_dangle"] } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index a01a5bb0a36..052eb55b408 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -5,27 +5,31 @@ pub use UnsafeSource::*; pub use crate::symbol::{Ident, Symbol as Name}; pub use crate::util::parser::ExprPrecedence; -use crate::ext::hygiene::{Mark, SyntaxContext}; +use crate::ext::hygiene::{ExpnId, SyntaxContext}; +use crate::parse::token::{self, DelimToken}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; -use crate::symbol::{keywords, Symbol}; +use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::TokenStream; use crate::ThinVec; use rustc_data_structures::indexed_vec::Idx; #[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert; +use rustc_data_structures::static_assert_size; use rustc_target::spec::abi::Abi; use syntax_pos::{Span, DUMMY_SP}; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::sync::Lrc; -use serialize::{self, Decoder, Encoder}; +use rustc_serialize::{self, Decoder, Encoder}; use std::fmt; pub use rustc_target::abi::FloatTy; +#[cfg(test)] +mod tests; + #[derive(Clone, RustcEncodable, RustcDecodable, Copy)] pub struct Label { pub ident: Ident, @@ -49,11 +53,17 @@ impl fmt::Debug for Lifetime { f, "lifetime({}: {})", self.id, - pprust::lifetime_to_string(self) + self ) } } +impl fmt::Display for Lifetime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.ident.name.as_str()) + } +} + /// A "Path" is essentially Rust's notion of a name. /// /// It's represented as a sequence of identifiers, @@ -64,13 +74,15 @@ impl fmt::Debug for Lifetime { pub struct Path { pub span: Span, /// The segments in the path: the things separated by `::`. - /// Global paths begin with `keywords::PathRoot`. + /// Global paths begin with `kw::PathRoot`. pub segments: Vec<PathSegment>, } -impl<'a> PartialEq<&'a str> for Path { - fn eq(&self, string: &&'a str) -> bool { - self.segments.len() == 1 && self.segments[0].ident.name == *string +impl PartialEq<Symbol> for Path { + fn eq(&self, symbol: &Symbol) -> bool { + self.segments.len() == 1 && { + self.segments[0].ident.name == *symbol + } } } @@ -97,7 +109,7 @@ impl Path { } pub fn is_global(&self) -> bool { - !self.segments.is_empty() && self.segments[0].ident.name == keywords::PathRoot.name() + !self.segments.is_empty() && self.segments[0].ident.name == kw::PathRoot } } @@ -125,7 +137,7 @@ impl PathSegment { PathSegment { ident, id: DUMMY_NODE_ID, args: None } } pub fn path_root(span: Span) -> Self { - PathSegment::from_ident(Ident::new(keywords::PathRoot.name(), span)) + PathSegment::from_ident(Ident::new(kw::PathRoot, span)) } } @@ -187,9 +199,9 @@ pub struct AngleBracketedArgs { pub span: Span, /// The arguments for this path segment. pub args: Vec<GenericArg>, - /// Bindings (equality constraints) on associated types, if present. - /// E.g., `Foo<A = Bar>`. - pub bindings: Vec<TypeBinding>, + /// Constraints on associated types, if any. + /// E.g., `Foo<A = Bar, B: Baz>`. + pub constraints: Vec<AssocTyConstraint>, } impl Into<Option<P<GenericArgs>>> for AngleBracketedArgs { @@ -210,7 +222,7 @@ pub struct ParenthesizedArgs { /// Overall span pub span: Span, - /// `(A,B)` + /// `(A, B)` pub inputs: Vec<P<Ty>>, /// `C` @@ -222,7 +234,7 @@ impl ParenthesizedArgs { AngleBracketedArgs { span: self.span, args: self.inputs.iter().cloned().map(|input| GenericArg::Type(input)).collect(), - bindings: vec![], + constraints: vec![], } } } @@ -242,12 +254,12 @@ mod node_id_inner { pub use node_id_inner::NodeId; impl NodeId { - pub fn placeholder_from_mark(mark: Mark) -> Self { - NodeId::from_u32(mark.as_u32()) + pub fn placeholder_from_expn_id(expn_id: ExpnId) -> Self { + NodeId::from_u32(expn_id.as_u32()) } - pub fn placeholder_to_mark(self) -> Mark { - Mark::from_u32(self.as_u32()) + pub fn placeholder_to_expn_id(self) -> ExpnId { + ExpnId::from_u32(self.as_u32()) } } @@ -257,13 +269,13 @@ impl fmt::Display for NodeId { } } -impl serialize::UseSpecializedEncodable for NodeId { +impl rustc_serialize::UseSpecializedEncodable for NodeId { fn default_encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { s.emit_u32(self.as_u32()) } } -impl serialize::UseSpecializedDecodable for NodeId { +impl rustc_serialize::UseSpecializedDecodable for NodeId { fn default_decode<D: Decoder>(d: &mut D) -> Result<NodeId, D::Error> { d.read_u32().map(NodeId::from_u32) } @@ -359,7 +371,6 @@ impl Default for Generics { Generics { params: Vec::new(), where_clause: WhereClause { - id: DUMMY_NODE_ID, predicates: Vec::new(), span: DUMMY_SP, }, @@ -371,7 +382,6 @@ impl Default for Generics { /// A where-clause in a definition. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct WhereClause { - pub id: NodeId, pub predicates: Vec<WherePredicate>, pub span: Span, } @@ -443,14 +453,11 @@ pub struct Crate { pub span: Span, } -/// A spanned compile-time attribute list item. -pub type NestedMetaItem = Spanned<NestedMetaItemKind>; - /// Possible values inside of compile-time attribute lists. /// /// E.g., the '..' in `#[name(..)]`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] -pub enum NestedMetaItemKind { +pub enum NestedMetaItem { /// A full MetaItem, for recursive meta items. MetaItem(MetaItem), /// A literal. @@ -464,7 +471,7 @@ pub enum NestedMetaItemKind { /// E.g., `#[test]`, `#[derive(..)]`, `#[rustfmt::skip]` or `#[feature = "foo"]`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct MetaItem { - pub ident: Path, + pub path: Path, pub node: MetaItemKind, pub span: Span, } @@ -515,21 +522,28 @@ impl fmt::Debug for Pat { } impl Pat { + /// Attempt reparsing the pattern as a type. + /// This is intended for use by diagnostics. pub(super) fn to_ty(&self) -> Option<P<Ty>> { let node = match &self.node { + // In a type expression `_` is an inference variable. PatKind::Wild => TyKind::Infer, + // An IDENT pattern with no binding mode would be valid as path to a type. E.g. `u32`. PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None) => { TyKind::Path(None, Path::from_ident(*ident)) } PatKind::Path(qself, path) => TyKind::Path(qself.clone(), path.clone()), PatKind::Mac(mac) => TyKind::Mac(mac.clone()), + // `&mut? P` can be reinterpreted as `&mut? T` where `T` is `P` reparsed as a type. PatKind::Ref(pat, mutbl) => pat .to_ty() .map(|ty| TyKind::Rptr(None, MutTy { ty, mutbl: *mutbl }))?, - PatKind::Slice(pats, None, _) if pats.len() == 1 => { - pats[0].to_ty().map(TyKind::Slice)? - } - PatKind::Tuple(pats, None) => { + // A slice/array pattern `[P]` can be reparsed as `[T]`, an unsized array, + // when `P` can be reparsed as a type `T`. + PatKind::Slice(pats) if pats.len() == 1 => pats[0].to_ty().map(TyKind::Slice)?, + // A tuple pattern `(P0, .., Pn)` can be reparsed as `(T0, .., Tn)` + // assuming `T0` to `Tn` are all syntactically valid as types. + PatKind::Tuple(pats) => { let mut tys = Vec::with_capacity(pats.len()); // FIXME(#48994) - could just be collected into an Option<Vec> for pat in pats { @@ -555,19 +569,15 @@ impl Pat { return false; } - match self.node { - PatKind::Ident(_, _, Some(ref p)) => p.walk(it), - PatKind::Struct(_, ref fields, _) => fields.iter().all(|field| field.node.pat.walk(it)), - PatKind::TupleStruct(_, ref s, _) | PatKind::Tuple(ref s, _) => { + match &self.node { + PatKind::Ident(_, _, Some(p)) => p.walk(it), + PatKind::Struct(_, fields, _) => fields.iter().all(|field| field.node.pat.walk(it)), + PatKind::TupleStruct(_, s) | PatKind::Tuple(s) | PatKind::Slice(s) => { s.iter().all(|p| p.walk(it)) } - PatKind::Box(ref s) | PatKind::Ref(ref s, _) | PatKind::Paren(ref s) => s.walk(it), - PatKind::Slice(ref before, ref slice, ref after) => { - before.iter().all(|p| p.walk(it)) - && slice.iter().all(|p| p.walk(it)) - && after.iter().all(|p| p.walk(it)) - } + PatKind::Box(s) | PatKind::Ref(s, _) | PatKind::Paren(s) => s.walk(it), PatKind::Wild + | PatKind::Rest | PatKind::Lit(_) | PatKind::Range(..) | PatKind::Ident(..) @@ -575,6 +585,14 @@ impl Pat { | PatKind::Mac(_) => true, } } + + /// Is this a `..` pattern? + pub fn is_rest(&self) -> bool { + match self.node { + PatKind::Rest => true, + _ => false, + } + } } /// A single field in a struct pattern @@ -623,12 +641,10 @@ pub enum PatKind { /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`). /// The `bool` is `true` in the presence of a `..`. - Struct(Path, Vec<Spanned<FieldPat>>, bool), + Struct(Path, Vec<Spanned<FieldPat>>, /* recovered */ bool), /// A tuple struct/variant pattern (`Variant(x, y, .., z)`). - /// If the `..` pattern fragment is present, then `Option<usize>` denotes its position. - /// `0 <= position <= subpats.len()`. - TupleStruct(Path, Vec<P<Pat>>, Option<usize>), + TupleStruct(Path, Vec<P<Pat>>), /// A possibly qualified path pattern. /// Unqualified path patterns `A::B::C` can legally refer to variants, structs, constants @@ -637,9 +653,7 @@ pub enum PatKind { Path(Option<QSelf>, Path), /// A tuple pattern (`(a, b)`). - /// If the `..` pattern fragment is present, then `Option<usize>` denotes its position. - /// `0 <= position <= subpats.len()`. - Tuple(Vec<P<Pat>>, Option<usize>), + Tuple(Vec<P<Pat>>), /// A `box` pattern. Box(P<Pat>), @@ -653,9 +667,22 @@ pub enum PatKind { /// A range pattern (e.g., `1...2`, `1..=2` or `1..2`). Range(P<Expr>, P<Expr>, Spanned<RangeEnd>), - /// `[a, b, ..i, y, z]` is represented as: - /// `PatKind::Slice(box [a, b], Some(i), box [y, z])` - Slice(Vec<P<Pat>>, Option<P<Pat>>, Vec<P<Pat>>), + /// A slice pattern `[a, b, c]`. + Slice(Vec<P<Pat>>), + + /// A rest pattern `..`. + /// + /// Syntactically it is valid anywhere. + /// + /// Semantically however, it only has meaning immediately inside: + /// - a slice pattern: `[a, .., b]`, + /// - a binding pattern immediately inside a slice pattern: `[a, r @ ..]`, + /// - a tuple pattern: `(a, .., b)`, + /// - a tuple struct/variant pattern: `$path(a, .., b)`. + /// + /// In all of these cases, an additional restriction applies, + /// only one rest pattern may occur in the pattern sequences. + Rest, /// Parentheses in patterns used for grouping (i.e., `(PAT)`). Paren(P<Pat>), @@ -895,13 +922,9 @@ pub struct Local { pub struct Arm { pub attrs: Vec<Attribute>, pub pats: Vec<P<Pat>>, - pub guard: Option<Guard>, + pub guard: Option<P<Expr>>, pub body: P<Expr>, -} - -#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] -pub enum Guard { - If(P<Expr>), + pub span: Span, } #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] @@ -949,7 +972,7 @@ pub struct Expr { // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_EXPR: std::mem::size_of::<Expr>() == 88); +static_assert_size!(Expr, 96); impl Expr { /// Whether this expression would be valid somewhere that expects a value; for example, an `if` @@ -1025,7 +1048,6 @@ impl Expr { pub fn precedence(&self) -> ExprPrecedence { match self.node { ExprKind::Box(_) => ExprPrecedence::Box, - ExprKind::ObsoleteInPlace(..) => ExprPrecedence::ObsoleteInPlace, ExprKind::Array(_) => ExprPrecedence::Array, ExprKind::Call(..) => ExprPrecedence::Call, ExprKind::MethodCall(..) => ExprPrecedence::MethodCall, @@ -1034,10 +1056,9 @@ impl Expr { ExprKind::Unary(..) => ExprPrecedence::Unary, ExprKind::Lit(_) => ExprPrecedence::Lit, ExprKind::Type(..) | ExprKind::Cast(..) => ExprPrecedence::Cast, + ExprKind::Let(..) => ExprPrecedence::Let, ExprKind::If(..) => ExprPrecedence::If, - ExprKind::IfLet(..) => ExprPrecedence::IfLet, ExprKind::While(..) => ExprPrecedence::While, - ExprKind::WhileLet(..) => ExprPrecedence::WhileLet, ExprKind::ForLoop(..) => ExprPrecedence::ForLoop, ExprKind::Loop(..) => ExprPrecedence::Loop, ExprKind::Match(..) => ExprPrecedence::Match, @@ -1045,6 +1066,7 @@ impl Expr { ExprKind::Block(..) => ExprPrecedence::Block, ExprKind::TryBlock(..) => ExprPrecedence::TryBlock, ExprKind::Async(..) => ExprPrecedence::Async, + ExprKind::Await(..) => ExprPrecedence::Await, ExprKind::Assign(..) => ExprPrecedence::Assign, ExprKind::AssignOp(..) => ExprPrecedence::AssignOp, ExprKind::Field(..) => ExprPrecedence::Field, @@ -1086,8 +1108,6 @@ pub enum RangeLimits { pub enum ExprKind { /// A `box x` expression. Box(P<Expr>), - /// First expr is the place; second expr is the value. - ObsoleteInPlace(P<Expr>, P<Expr>), /// An array (`[a, b, c, d]`) Array(Vec<P<Expr>>), /// A function call @@ -1117,27 +1137,22 @@ pub enum ExprKind { Lit(Lit), /// A cast (e.g., `foo as f64`). Cast(P<Expr>, P<Ty>), + /// A type ascription (e.g., `42: usize`). Type(P<Expr>, P<Ty>), + /// A `let pats = expr` expression that is only semantically allowed in the condition + /// of `if` / `while` expressions. (e.g., `if let 0 = x { .. }`). + /// + /// The `Vec<P<Pat>>` is for or-patterns at the top level. + /// FIXME(54883): Change this to just `P<Pat>`. + Let(Vec<P<Pat>>, P<Expr>), /// An `if` block, with an optional `else` block. /// /// `if expr { block } else { expr }` If(P<Expr>, P<Block>, Option<P<Expr>>), - /// An `if let` expression with an optional else block - /// - /// `if let pat = expr { block } else { expr }` - /// - /// This is desugared to a `match` expression. - IfLet(Vec<P<Pat>>, P<Expr>, P<Block>, Option<P<Expr>>), - /// A while loop, with an optional label + /// A while loop, with an optional label. /// /// `'label: while expr { block }` While(P<Expr>, P<Block>, Option<Label>), - /// A `while let` loop, with an optional label. - /// - /// `'label: while let pat = expr { block }` - /// - /// This is desugared to a combination of `loop` and `match` expressions. - WhileLet(Vec<P<Pat>>, P<Expr>, P<Block>, Option<Label>), /// A `for` loop, with an optional label. /// /// `'label: for pat in expr { block }` @@ -1165,6 +1180,9 @@ pub enum ExprKind { /// created during lowering cannot be made the parent of any other /// preexisting defs. Async(CaptureBy, NodeId, P<Block>), + /// An await expression (`my_future.await`). + Await(P<Expr>), + /// A try block (`try { ... }`). TryBlock(P<Block>), @@ -1178,7 +1196,7 @@ pub enum ExprKind { Field(P<Expr>, Ident), /// An indexing operation (e.g., `foo[2]`). Index(P<Expr>, P<Expr>), - /// A range (e.g., `1..2`, `1..`, `..2`, `1...2`, `1...`, `...2`). + /// A range (e.g., `1..2`, `1..`, `..2`, `1..=2`, `..=2`). Range(Option<P<Expr>>, Option<P<Expr>>, RangeLimits), /// Variable reference, possibly containing `::` and/or type @@ -1279,6 +1297,7 @@ pub struct Mac_ { pub path: Path, pub delim: MacDelimiter, pub tts: TokenStream, + pub prior_type_ascription: Option<(Span, bool)>, } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Debug)] @@ -1294,6 +1313,16 @@ impl Mac_ { } } +impl MacDelimiter { + crate fn to_token(self) -> DelimToken { + match self { + MacDelimiter::Parenthesis => DelimToken::Paren, + MacDelimiter::Bracket => DelimToken::Bracket, + MacDelimiter::Brace => DelimToken::Brace, + } + } +} + #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct MacroDef { pub tokens: TokenStream, @@ -1316,8 +1345,17 @@ pub enum StrStyle { Raw(u16), } -/// A literal. -pub type Lit = Spanned<LitKind>; +/// An AST literal. +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] +pub struct Lit { + /// The original literal token as written in source code. + pub token: token::Lit, + /// The "semantic" representation of the literal lowered from the original tokens. + /// Strings are unescaped, hexadecimal forms are eliminated, etc. + /// FIXME: Remove this and only create the semantic representation during lowering to HIR. + pub node: LitKind, + pub span: Span, +} #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Copy, Hash, PartialEq)] pub enum LitIntType { @@ -1347,7 +1385,7 @@ pub enum LitKind { FloatUnsuffixed(Symbol), /// A boolean literal. Bool(bool), - /// A recovered character literal that contains mutliple `char`s, most likely a typo. + /// Placeholder for a literal that wasn't well-formed in some way. Err(Symbol), } @@ -1385,10 +1423,10 @@ impl LitKind { | LitKind::ByteStr(..) | LitKind::Byte(..) | LitKind::Char(..) - | LitKind::Err(..) | LitKind::Int(_, LitIntType::Unsuffixed) | LitKind::FloatUnsuffixed(..) - | LitKind::Bool(..) => true, + | LitKind::Bool(..) + | LitKind::Err(..) => true, // suffixed variants LitKind::Int(_, LitIntType::Signed(..)) | LitKind::Int(_, LitIntType::Unsigned(..)) @@ -1442,6 +1480,7 @@ pub enum TraitItemKind { Macro(Mac), } +/// Represents anything within an `impl` block. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct ImplItem { pub id: NodeId, @@ -1456,12 +1495,13 @@ pub struct ImplItem { pub tokens: Option<TokenStream>, } +/// Represents various kinds of content within an `impl`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub enum ImplItemKind { Const(P<Ty>, P<Expr>), Method(MethodSig, P<Block>), - Type(P<Ty>), - Existential(GenericBounds), + TyAlias(P<Ty>), + OpaqueTy(GenericBounds), Macro(Mac), } @@ -1499,6 +1539,17 @@ impl IntTy { } } + pub fn to_symbol(&self) -> Symbol { + match *self { + IntTy::Isize => sym::isize, + IntTy::I8 => sym::i8, + IntTy::I16 => sym::i16, + IntTy::I32 => sym::i32, + IntTy::I64 => sym::i64, + IntTy::I128 => sym::i128, + } + } + pub fn val_to_string(&self, val: i128) -> String { // Cast to a `u128` so we can correctly print `INT128_MIN`. All integral types // are parsed as `u128`, so we wouldn't want to print an extra negative @@ -1540,6 +1591,17 @@ impl UintTy { } } + pub fn to_symbol(&self) -> Symbol { + match *self { + UintTy::Usize => sym::usize, + UintTy::U8 => sym::u8, + UintTy::U16 => sym::u16, + UintTy::U32 => sym::u32, + UintTy::U64 => sym::u64, + UintTy::U128 => sym::u128, + } + } + pub fn val_to_string(&self, val: u128) -> String { format!("{}{}", val, self.ty_to_string()) } @@ -1568,15 +1630,29 @@ impl fmt::Display for UintTy { } } -// Bind a type to an associated type: `A = Foo`. +/// A constraint on an associated type (e.g., `A = Bar` in `Foo<A = Bar>` or +/// `A: TraitA + TraitB` in `Foo<A: TraitA + TraitB>`). #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] -pub struct TypeBinding { +pub struct AssocTyConstraint { pub id: NodeId, pub ident: Ident, - pub ty: P<Ty>, + pub kind: AssocTyConstraintKind, pub span: Span, } +/// The kinds of an `AssocTyConstraint`. +#[derive(Clone, RustcEncodable, RustcDecodable, Debug)] +pub enum AssocTyConstraintKind { + /// E.g., `A = Bar` in `Foo<A = Bar>`. + Equality { + ty: P<Ty>, + }, + /// E.g. `A: TraitA + TraitB` in `Foo<A: TraitA + TraitB>`. + Bound { + bounds: GenericBounds, + }, +} + #[derive(Clone, RustcEncodable, RustcDecodable)] pub struct Ty { pub id: NodeId, @@ -1628,7 +1704,7 @@ pub enum TyKind { /// /// The `NodeId` exists to prevent lowering from having to /// generate `NodeId`s on the fly, which would complicate - /// the generation of `existential type` items significantly. + /// the generation of opaque `type Foo = impl Trait` items significantly. ImplTrait(NodeId, GenericBounds), /// No-op; kept solely so that we can pretty-print faithfully. Paren(P<Ty>), @@ -1713,9 +1789,11 @@ pub struct InlineAsm { /// E.g., `bar: usize` as in `fn foo(bar: usize)`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Arg { + pub attrs: ThinVec<Attribute>, pub ty: P<Ty>, pub pat: P<Pat>, pub id: NodeId, + pub span: Span, } /// Alternative representation for `Arg`s describing `self` parameter of methods. @@ -1736,7 +1814,7 @@ pub type ExplicitSelf = Spanned<SelfKind>; impl Arg { pub fn to_self(&self) -> Option<ExplicitSelf> { if let PatKind::Ident(BindingMode::ByValue(mutbl), ident, _) = self.pat.node { - if ident.name == keywords::SelfLower.name() { + if ident.name == kw::SelfLower { return match self.ty.node { TyKind::ImplicitSelf => Some(respan(self.pat.span, SelfKind::Value(mutbl))), TyKind::Rptr(lt, MutTy { ref ty, mutbl }) if ty.node.is_implicit_self() => { @@ -1754,13 +1832,13 @@ impl Arg { pub fn is_self(&self) -> bool { if let PatKind::Ident(_, ident, _) = self.pat.node { - ident.name == keywords::SelfLower.name() + ident.name == kw::SelfLower } else { false } } - pub fn from_self(eself: ExplicitSelf, eself_ident: Ident) -> Arg { + pub fn from_self(attrs: ThinVec<Attribute>, eself: ExplicitSelf, eself_ident: Ident) -> Arg { let span = eself.span.to(eself_ident.span); let infer_ty = P(Ty { id: DUMMY_NODE_ID, @@ -1768,11 +1846,13 @@ impl Arg { span, }); let arg = |mutbl, ty| Arg { + attrs, pat: P(Pat { id: DUMMY_NODE_ID, node: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None), span, }), + span, ty, id: DUMMY_NODE_ID, }; @@ -1787,7 +1867,7 @@ impl Arg { lt, MutTy { ty: infer_ty, - mutbl: mutbl, + mutbl, }, ), span, @@ -1797,7 +1877,7 @@ impl Arg { } } -/// Header (not the body) of a function declaration. +/// A header (not the body) of a function declaration. /// /// E.g., `fn foo(bar: baz)`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] @@ -1959,8 +2039,13 @@ pub struct EnumDef { #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct Variant_ { + /// Name of the variant. pub ident: Ident, + /// Attributes of the variant. pub attrs: Vec<Attribute>, + /// Id of the variant (not the constructor, see `VariantData::ctor_id()`). + pub id: NodeId, + /// Fields and constructor id of the variant. pub data: VariantData, /// Explicit discriminant, e.g., `Foo = 1`. pub disr_expr: Option<AnonConst>, @@ -2016,9 +2101,7 @@ pub enum AttrStyle { Inner, } -#[derive( - Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, PartialOrd, Ord, Copy, -)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord, Copy)] pub struct AttrId(pub usize); impl Idx for AttrId { @@ -2030,6 +2113,18 @@ impl Idx for AttrId { } } +impl rustc_serialize::Encodable for AttrId { + fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> { + s.emit_unit() + } +} + +impl rustc_serialize::Decodable for AttrId { + fn decode<D: Decoder>(d: &mut D) -> Result<AttrId, D::Error> { + d.read_nil().map(|_| crate::attr::mk_attr_id()) + } +} + /// Metadata associated with an item. /// Doc-comments are promoted to attributes that have `is_sugared_doc = true`. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] @@ -2056,10 +2151,10 @@ pub struct TraitRef { #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub struct PolyTraitRef { - /// The `'a` in `<'a> Foo<&'a T>` + /// The `'a` in `<'a> Foo<&'a T>`. pub bound_generic_params: Vec<GenericParam>, - /// The `Foo<&'a T>` in `<'a> Foo<&'a T>` + /// The `Foo<&'a T>` in `<'a> Foo<&'a T>`. pub trait_ref: TraitRef, pub span: Span, @@ -2070,7 +2165,7 @@ impl PolyTraitRef { PolyTraitRef { bound_generic_params: generic_params, trait_ref: TraitRef { - path: path, + path, ref_id: DUMMY_NODE_ID, }, span, @@ -2120,23 +2215,13 @@ pub struct StructField { pub attrs: Vec<Attribute>, } -/// Fields and Ids of enum variants and structs -/// -/// For enum variants: `NodeId` represents both an Id of the variant itself (relevant for all -/// variant kinds) and an Id of the variant's constructor (not relevant for `Struct`-variants). -/// One shared Id can be successfully used for these two purposes. -/// Id of the whole enum lives in `Item`. -/// -/// For structs: `NodeId` represents an Id of the structure's constructor, so it is not actually -/// used for `Struct`-structs (but still presents). Structures don't have an analogue of "Id of -/// the variant itself" from enum variants. -/// Id of the whole struct lives in `Item`. +/// Fields and constructor ids of enum variants and structs. #[derive(Clone, RustcEncodable, RustcDecodable, Debug)] pub enum VariantData { /// Struct variant. /// /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`. - Struct(Vec<StructField>, NodeId), + Struct(Vec<StructField>, bool), /// Tuple variant. /// /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`. @@ -2148,36 +2233,19 @@ pub enum VariantData { } impl VariantData { + /// Return the fields of this variant. pub fn fields(&self) -> &[StructField] { match *self { - VariantData::Struct(ref fields, _) | VariantData::Tuple(ref fields, _) => fields, + VariantData::Struct(ref fields, ..) | VariantData::Tuple(ref fields, _) => fields, _ => &[], } } - pub fn id(&self) -> NodeId { + + /// Return the `NodeId` of this variant's constructor, if it has one. + pub fn ctor_id(&self) -> Option<NodeId> { match *self { - VariantData::Struct(_, id) | VariantData::Tuple(_, id) | VariantData::Unit(id) => id, - } - } - pub fn is_struct(&self) -> bool { - if let VariantData::Struct(..) = *self { - true - } else { - false - } - } - pub fn is_tuple(&self) -> bool { - if let VariantData::Tuple(..) = *self { - true - } else { - false - } - } - pub fn is_unit(&self) -> bool { - if let VariantData::Unit(..) = *self { - true - } else { - false + VariantData::Struct(..) => None, + VariantData::Tuple(_, id) | VariantData::Unit(id) => Some(id), } } } @@ -2207,7 +2275,7 @@ pub struct Item { impl Item { /// Return the span that encompasses the attributes. pub fn span_with_attributes(&self) -> Span { - self.attrs.iter().fold(self.span, |acc, attr| acc.to(attr.span())) + self.attrs.iter().fold(self.span, |acc, attr| acc.to(attr.span)) } } @@ -2269,11 +2337,11 @@ pub enum ItemKind { /// A type alias (`type` or `pub type`). /// /// E.g., `type Foo = Bar<u8>;`. - Ty(P<Ty>, Generics), - /// An existential type declaration (`existential type`). + TyAlias(P<Ty>, Generics), + /// An opaque `impl Trait` type alias. /// - /// E.g., `existential type Foo: Bar + Boo;`. - Existential(GenericBounds, Generics), + /// E.g., `type Foo = impl Bar + Boo;`. + OpaqueTy(GenericBounds, Generics), /// An enum definition (`enum` or `pub enum`). /// /// E.g., `enum Foo<A, B> { C<A>, D<B> }`. @@ -2326,8 +2394,8 @@ impl ItemKind { ItemKind::Mod(..) => "module", ItemKind::ForeignMod(..) => "foreign module", ItemKind::GlobalAsm(..) => "global asm", - ItemKind::Ty(..) => "type alias", - ItemKind::Existential(..) => "existential type", + ItemKind::TyAlias(..) => "type alias", + ItemKind::OpaqueTy(..) => "opaque type", ItemKind::Enum(..) => "enum", ItemKind::Struct(..) => "struct", ItemKind::Union(..) => "union", @@ -2353,9 +2421,8 @@ pub struct ForeignItem { pub enum ForeignItemKind { /// A foreign function. Fn(P<FnDecl>, Generics), - /// A foreign static item (`static ext: u8`), with optional mutability. - /// (The boolean is `true` for mutable items). - Static(P<Ty>, bool), + /// A foreign static item (`static ext: u8`). + Static(P<Ty>, Mutability), /// A foreign type. Ty, /// A macro invocation. @@ -2372,16 +2439,3 @@ impl ForeignItemKind { } } } - -#[cfg(test)] -mod tests { - use super::*; - use serialize; - - // Are ASTs encodable? - #[test] - fn check_asts_encodable() { - fn assert_encodable<T: serialize::Encodable>() {} - assert_encodable::<Crate>(); - } -} diff --git a/src/libsyntax/ast/tests.rs b/src/libsyntax/ast/tests.rs new file mode 100644 index 00000000000..7558e9cc3a3 --- /dev/null +++ b/src/libsyntax/ast/tests.rs @@ -0,0 +1,8 @@ +use super::*; + +// Are ASTs encodable? +#[test] +fn check_asts_encodable() { + fn assert_encodable<T: rustc_serialize::Encodable>() {} + assert_encodable::<Crate>(); +} diff --git a/src/libsyntax/attr/builtin.rs b/src/libsyntax/attr/builtin.rs index f7a000935ca..5fb513783fb 100644 --- a/src/libsyntax/attr/builtin.rs +++ b/src/libsyntax/attr/builtin.rs @@ -1,23 +1,47 @@ //! Parsing and validation of builtin attributes -use crate::ast::{self, Attribute, MetaItem, Name, NestedMetaItemKind}; +use crate::ast::{self, Attribute, MetaItem, NestedMetaItem}; +use crate::early_buffered_lints::BufferedEarlyLintId; +use crate::ext::base::ExtCtxt; use crate::feature_gate::{Features, GatedCfg}; use crate::parse::ParseSess; use errors::{Applicability, Handler}; -use syntax_pos::{symbol::Symbol, Span}; +use syntax_pos::hygiene::Transparency; +use syntax_pos::{symbol::Symbol, symbol::sym, Span}; use super::{mark_used, MetaItemKind}; enum AttrError { - MultipleItem(Name), - UnknownMetaItem(Name, &'static [&'static str]), + MultipleItem(String), + UnknownMetaItem(String, &'static [&'static str]), MissingSince, MissingFeature, MultipleStabilityLevels, UnsupportedLiteral(&'static str, /* is_bytestr */ bool), } +/// A template that the attribute input must match. +/// Only top-level shape (`#[attr]` vs `#[attr(...)]` vs `#[attr = ...]`) is considered now. +#[derive(Clone, Copy)] +pub struct AttributeTemplate { + crate word: bool, + crate list: Option<&'static str>, + crate name_value_str: Option<&'static str>, +} + +impl AttributeTemplate { + /// Checks that the given meta-item is compatible with this template. + fn compatible(&self, meta_item_kind: &ast::MetaItemKind) -> bool { + match meta_item_kind { + ast::MetaItemKind::Word => self.word, + ast::MetaItemKind::List(..) => self.list.is_some(), + ast::MetaItemKind::NameValue(lit) if lit.node.is_str() => self.name_value_str.is_some(), + ast::MetaItemKind::NameValue(..) => false, + } + } +} + fn handle_errors(sess: &ParseSess, span: Span, error: AttrError) { let diag = &sess.span_diagnostic; match error { @@ -80,19 +104,27 @@ pub enum UnwindAttr { /// Determine what `#[unwind]` attribute is present in `attrs`, if any. pub fn find_unwind_attr(diagnostic: Option<&Handler>, attrs: &[Attribute]) -> Option<UnwindAttr> { attrs.iter().fold(None, |ia, attr| { - if attr.check_name("unwind") { + if attr.check_name(sym::unwind) { if let Some(meta) = attr.meta() { if let MetaItemKind::List(items) = meta.node { if items.len() == 1 { - if items[0].check_name("allowed") { + if items[0].check_name(sym::allowed) { return Some(UnwindAttr::Allowed); - } else if items[0].check_name("aborts") { + } else if items[0].check_name(sym::aborts) { return Some(UnwindAttr::Aborts); } } diagnostic.map(|d| { - span_err!(d, attr.span, E0633, "malformed `#[unwind]` attribute"); + struct_span_err!(d, attr.span, E0633, "malformed `unwind` attribute input") + .span_label(attr.span, "invalid argument") + .span_suggestions( + attr.span, + "the allowed arguments are `allowed` and `aborts`", + (vec!["allowed", "aborts"]).into_iter() + .map(|s| format!("#[unwind({})]", s)), + Applicability::MachineApplicable, + ).emit(); }); } } @@ -103,7 +135,7 @@ pub fn find_unwind_attr(diagnostic: Option<&Handler>, attrs: &[Attribute]) -> Op } /// Represents the #[stable], #[unstable], #[rustc_{deprecated,const_unstable}] attributes. -#[derive(RustcEncodable, RustcDecodable, Clone, Debug, PartialEq, Eq, Hash)] +#[derive(RustcEncodable, RustcDecodable, Copy, Clone, Debug, PartialEq, Eq, Hash)] pub struct Stability { pub level: StabilityLevel, pub feature: Symbol, @@ -114,16 +146,31 @@ pub struct Stability { pub const_stability: Option<Symbol>, /// whether the function has a `#[rustc_promotable]` attribute pub promotable: bool, + /// whether the function has a `#[rustc_allow_const_fn_ptr]` attribute + pub allow_const_fn_ptr: bool, } /// The available stability levels. -#[derive(RustcEncodable, RustcDecodable, PartialEq, PartialOrd, Clone, Debug, Eq, Hash)] +#[derive(RustcEncodable, RustcDecodable, PartialEq, PartialOrd, Copy, Clone, Debug, Eq, Hash)] pub enum StabilityLevel { // Reason for the current stability level and the relevant rust-lang issue Unstable { reason: Option<Symbol>, issue: u32 }, Stable { since: Symbol }, } +impl Stability { + pub fn unstable(feature: Symbol, reason: Option<Symbol>, issue: u32) -> Stability { + Stability { + level: StabilityLevel::Unstable { reason, issue }, + feature, + rustc_depr: None, + const_stability: None, + promotable: false, + allow_const_fn_ptr: false, + } + } +} + impl StabilityLevel { pub fn is_unstable(&self) -> bool { if let StabilityLevel::Unstable {..} = *self { @@ -141,7 +188,7 @@ impl StabilityLevel { } } -#[derive(RustcEncodable, RustcDecodable, PartialEq, PartialOrd, Clone, Debug, Eq, Hash)] +#[derive(RustcEncodable, RustcDecodable, PartialEq, PartialOrd, Copy, Clone, Debug, Eq, Hash)] pub struct RustcDeprecation { pub since: Symbol, pub reason: Symbol, @@ -151,19 +198,17 @@ pub struct RustcDeprecation { /// Checks if `attrs` contains an attribute like `#![feature(feature_name)]`. /// This will not perform any "sanity checks" on the form of the attributes. -pub fn contains_feature_attr(attrs: &[Attribute], feature_name: &str) -> bool { +pub fn contains_feature_attr(attrs: &[Attribute], feature_name: Symbol) -> bool { attrs.iter().any(|item| { - item.check_name("feature") && + item.check_name(sym::feature) && item.meta_item_list().map(|list| { - list.iter().any(|mi| { - mi.word().map(|w| w.name() == feature_name) - .unwrap_or(false) - }) + list.iter().any(|mi| mi.is_word() && mi.check_name(feature_name)) }).unwrap_or(false) }) } -/// Finds the first stability attribute. `None` if none exists. +/// Collects stability info from all stability attributes in `attrs`. +/// Returns `None` if no stability attributes are found. pub fn find_stability(sess: &ParseSess, attrs: &[Attribute], item_sp: Span) -> Option<Stability> { find_stability_generic(sess, attrs.iter(), item_sp) @@ -181,15 +226,17 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, let mut rustc_depr: Option<RustcDeprecation> = None; let mut rustc_const_unstable: Option<Symbol> = None; let mut promotable = false; + let mut allow_const_fn_ptr = false; let diagnostic = &sess.span_diagnostic; 'outer: for attr in attrs_iter { if ![ - "rustc_deprecated", - "rustc_const_unstable", - "unstable", - "stable", - "rustc_promotable", + sym::rustc_deprecated, + sym::rustc_const_unstable, + sym::unstable, + sym::stable, + sym::rustc_promotable, + sym::rustc_allow_const_fn_ptr, ].iter().any(|&s| attr.path == s) { continue // not a stability level } @@ -198,15 +245,18 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, let meta = attr.meta(); - if attr.path == "rustc_promotable" { + if attr.path == sym::rustc_promotable { promotable = true; } + if attr.path == sym::rustc_allow_const_fn_ptr { + allow_const_fn_ptr = true; + } // attributes with data else if let Some(MetaItem { node: MetaItemKind::List(ref metas), .. }) = meta { let meta = meta.as_ref().unwrap(); let get = |meta: &MetaItem, item: &mut Option<Symbol>| { if item.is_some() { - handle_errors(sess, meta.span, AttrError::MultipleItem(meta.name())); + handle_errors(sess, meta.span, AttrError::MultipleItem(meta.path.to_string())); return false } if let Some(v) = meta.value_str() { @@ -225,17 +275,16 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, )+ for meta in metas { if let Some(mi) = meta.meta_item() { - match &*mi.name().as_str() { + match mi.name_or_empty() { $( - stringify!($name) - => if !get(mi, &mut $name) { continue 'outer }, + sym::$name => if !get(mi, &mut $name) { continue 'outer }, )+ _ => { let expected = &[ $( stringify!($name) ),+ ]; handle_errors( sess, mi.span, - AttrError::UnknownMetaItem(mi.name(), expected), + AttrError::UnknownMetaItem(mi.path.to_string(), expected), ); continue 'outer } @@ -243,7 +292,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, } else { handle_errors( sess, - meta.span, + meta.span(), AttrError::UnsupportedLiteral( "unsupported literal", false, @@ -255,8 +304,8 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, } } - match &*meta.name().as_str() { - "rustc_deprecated" => { + match meta.name_or_empty() { + sym::rustc_deprecated => { if rustc_depr.is_some() { span_err!(diagnostic, item_sp, E0540, "multiple rustc_deprecated attributes"); @@ -274,16 +323,16 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, }) } (None, _) => { - handle_errors(sess, attr.span(), AttrError::MissingSince); + handle_errors(sess, attr.span, AttrError::MissingSince); continue } _ => { - span_err!(diagnostic, attr.span(), E0543, "missing 'reason'"); + span_err!(diagnostic, attr.span, E0543, "missing 'reason'"); continue } } } - "rustc_const_unstable" => { + sym::rustc_const_unstable => { if rustc_const_unstable.is_some() { span_err!(diagnostic, item_sp, E0553, "multiple rustc_const_unstable attributes"); @@ -294,13 +343,13 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, if let Some(feature) = feature { rustc_const_unstable = Some(feature); } else { - span_err!(diagnostic, attr.span(), E0629, "missing 'feature'"); + span_err!(diagnostic, attr.span, E0629, "missing 'feature'"); continue } } - "unstable" => { + sym::unstable => { if stab.is_some() { - handle_errors(sess, attr.span(), AttrError::MultipleStabilityLevels); + handle_errors(sess, attr.span, AttrError::MultipleStabilityLevels); break } @@ -309,16 +358,16 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, let mut issue = None; for meta in metas { if let Some(mi) = meta.meta_item() { - match &*mi.name().as_str() { - "feature" => if !get(mi, &mut feature) { continue 'outer }, - "reason" => if !get(mi, &mut reason) { continue 'outer }, - "issue" => if !get(mi, &mut issue) { continue 'outer }, + match mi.name_or_empty() { + sym::feature => if !get(mi, &mut feature) { continue 'outer }, + sym::reason => if !get(mi, &mut reason) { continue 'outer }, + sym::issue => if !get(mi, &mut issue) { continue 'outer }, _ => { handle_errors( sess, - meta.span, + meta.span(), AttrError::UnknownMetaItem( - mi.name(), + mi.path.to_string(), &["feature", "reason", "issue"] ), ); @@ -328,7 +377,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, } else { handle_errors( sess, - meta.span, + meta.span(), AttrError::UnsupportedLiteral( "unsupported literal", false, @@ -347,7 +396,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, if let Ok(issue) = issue.as_str().parse() { issue } else { - span_err!(diagnostic, attr.span(), E0545, + span_err!(diagnostic, attr.span, E0545, "incorrect 'issue'"); continue } @@ -357,45 +406,46 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, rustc_depr: None, const_stability: None, promotable: false, + allow_const_fn_ptr: false, }) } (None, _, _) => { - handle_errors(sess, attr.span(), AttrError::MissingFeature); + handle_errors(sess, attr.span, AttrError::MissingFeature); continue } _ => { - span_err!(diagnostic, attr.span(), E0547, "missing 'issue'"); + span_err!(diagnostic, attr.span, E0547, "missing 'issue'"); continue } } } - "stable" => { + sym::stable => { if stab.is_some() { - handle_errors(sess, attr.span(), AttrError::MultipleStabilityLevels); + handle_errors(sess, attr.span, AttrError::MultipleStabilityLevels); break } let mut feature = None; let mut since = None; for meta in metas { - match &meta.node { - NestedMetaItemKind::MetaItem(mi) => { - match &*mi.name().as_str() { - "feature" => if !get(mi, &mut feature) { continue 'outer }, - "since" => if !get(mi, &mut since) { continue 'outer }, + match meta { + NestedMetaItem::MetaItem(mi) => { + match mi.name_or_empty() { + sym::feature => if !get(mi, &mut feature) { continue 'outer }, + sym::since => if !get(mi, &mut since) { continue 'outer }, _ => { handle_errors( sess, - meta.span, + meta.span(), AttrError::UnknownMetaItem( - mi.name(), &["since", "note"], + mi.path.to_string(), &["since", "note"], ), ); continue 'outer } } }, - NestedMetaItemKind::Literal(lit) => { + NestedMetaItem::Literal(lit) => { handle_errors( sess, lit.span, @@ -419,14 +469,15 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, rustc_depr: None, const_stability: None, promotable: false, + allow_const_fn_ptr: false, }) } (None, _) => { - handle_errors(sess, attr.span(), AttrError::MissingFeature); + handle_errors(sess, attr.span, AttrError::MissingFeature); continue } _ => { - handle_errors(sess, attr.span(), AttrError::MissingSince); + handle_errors(sess, attr.span, AttrError::MissingSince); continue } } @@ -459,13 +510,14 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, } // Merge the const-unstable info into the stability info - if promotable { + if promotable || allow_const_fn_ptr { if let Some(ref mut stab) = stab { - stab.promotable = true; + stab.promotable = promotable; + stab.allow_const_fn_ptr = allow_const_fn_ptr; } else { span_err!(diagnostic, item_sp, E0717, - "rustc_promotable attribute must be paired with \ - either stable or unstable attribute"); + "rustc_promotable and rustc_allow_const_fn_ptr attributes \ + must be paired with either stable or unstable attribute"); } } @@ -473,7 +525,7 @@ fn find_stability_generic<'a, I>(sess: &ParseSess, } pub fn find_crate_name(attrs: &[Attribute]) -> Option<Symbol> { - super::first_attr_value_str_by_name(attrs, "crate_name") + super::first_attr_value_str_by_name(attrs, sym::crate_name) } /// Tests if a cfg-pattern matches the cfg set @@ -483,8 +535,8 @@ pub fn cfg_matches(cfg: &ast::MetaItem, sess: &ParseSess, features: Option<&Feat gated_cfg.check_and_emit(sess, feats); } let error = |span, msg| { sess.span_diagnostic.span_err(span, msg); true }; - if cfg.ident.segments.len() != 1 { - return error(cfg.ident.span, "`cfg` predicate key must be an identifier"); + if cfg.path.segments.len() != 1 { + return error(cfg.path.span, "`cfg` predicate key must be an identifier"); } match &cfg.node { MetaItemKind::List(..) => { @@ -502,7 +554,8 @@ pub fn cfg_matches(cfg: &ast::MetaItem, sess: &ParseSess, features: Option<&Feat true } MetaItemKind::NameValue(..) | MetaItemKind::Word => { - sess.config.contains(&(cfg.name(), cfg.value_str())) + let ident = cfg.ident().expect("multi-segment cfg predicate"); + sess.config.contains(&(ident.name, cfg.value_str())) } } }) @@ -520,7 +573,7 @@ pub fn eval_condition<F>(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) if !mi.is_meta_item() { handle_errors( sess, - mi.span, + mi.span(), AttrError::UnsupportedLiteral( "unsupported literal", false @@ -532,14 +585,14 @@ pub fn eval_condition<F>(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) // The unwraps below may look dangerous, but we've already asserted // that they won't fail with the loop above. - match &*cfg.name().as_str() { - "any" => mis.iter().any(|mi| { + match cfg.name_or_empty() { + sym::any => mis.iter().any(|mi| { eval_condition(mi.meta_item().unwrap(), sess, eval) }), - "all" => mis.iter().all(|mi| { + sym::all => mis.iter().all(|mi| { eval_condition(mi.meta_item().unwrap(), sess, eval) }), - "not" => { + sym::not => { if mis.len() != 1 { span_err!(sess.span_diagnostic, cfg.span, E0536, "expected 1 cfg-pattern"); return false; @@ -547,8 +600,9 @@ pub fn eval_condition<F>(cfg: &ast::MetaItem, sess: &ParseSess, eval: &mut F) !eval_condition(mis[0].meta_item().unwrap(), sess, eval) }, - p => { - span_err!(sess.span_diagnostic, cfg.span, E0537, "invalid predicate `{}`", p); + _ => { + span_err!(sess.span_diagnostic, cfg.span, E0537, + "invalid predicate `{}`", cfg.path); false } } @@ -582,7 +636,7 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, let diagnostic = &sess.span_diagnostic; 'outer: for attr in attrs_iter { - if !attr.check_name("deprecated") { + if !attr.check_name(sym::deprecated) { continue; } @@ -602,7 +656,9 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, MetaItemKind::List(list) => { let get = |meta: &MetaItem, item: &mut Option<Symbol>| { if item.is_some() { - handle_errors(sess, meta.span, AttrError::MultipleItem(meta.name())); + handle_errors( + sess, meta.span, AttrError::MultipleItem(meta.path.to_string()) + ); return false } if let Some(v) = meta.value_str() { @@ -630,22 +686,23 @@ fn find_deprecation_generic<'a, I>(sess: &ParseSess, let mut since = None; let mut note = None; for meta in list { - match &meta.node { - NestedMetaItemKind::MetaItem(mi) => { - match &*mi.name().as_str() { - "since" => if !get(mi, &mut since) { continue 'outer }, - "note" => if !get(mi, &mut note) { continue 'outer }, + match meta { + NestedMetaItem::MetaItem(mi) => { + match mi.name_or_empty() { + sym::since => if !get(mi, &mut since) { continue 'outer }, + sym::note => if !get(mi, &mut note) { continue 'outer }, _ => { handle_errors( sess, - meta.span, - AttrError::UnknownMetaItem(mi.name(), &["since", "note"]), + meta.span(), + AttrError::UnknownMetaItem(mi.path.to_string(), + &["since", "note"]), ); continue 'outer } } } - NestedMetaItemKind::Literal(lit) => { + NestedMetaItem::Literal(lit) => { handle_errors( sess, lit.span, @@ -707,14 +764,14 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { let mut acc = Vec::new(); let diagnostic = &sess.span_diagnostic; - if attr.path == "repr" { + if attr.path == sym::repr { if let Some(items) = attr.meta_item_list() { mark_used(attr); for item in items { if !item.is_meta_item() { handle_errors( sess, - item.span, + item.span(), AttrError::UnsupportedLiteral( "meta item in `repr` must be an identifier", false, @@ -724,19 +781,13 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { } let mut recognised = false; - if let Some(mi) = item.word() { - let word = &*mi.name().as_str(); - let hint = match word { - "C" => Some(ReprC), - "packed" => Some(ReprPacked(1)), - "simd" => Some(ReprSimd), - "transparent" => Some(ReprTransparent), - _ => match int_type_of_word(word) { - Some(ity) => Some(ReprInt(ity)), - None => { - None - } - } + if item.is_word() { + let hint = match item.name_or_empty() { + sym::C => Some(ReprC), + sym::packed => Some(ReprPacked(1)), + sym::simd => Some(ReprSimd), + sym::transparent => Some(ReprTransparent), + name => int_type_of_word(name).map(ReprInt), }; if let Some(h) = hint { @@ -762,14 +813,14 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { }; let mut literal_error = None; - if name == "align" { + if name == sym::align { recognised = true; match parse_alignment(&value.node) { Ok(literal) => acc.push(ReprAlign(literal)), Err(message) => literal_error = Some(message) }; } - else if name == "packed" { + else if name == sym::packed { recognised = true; match parse_alignment(&value.node) { Ok(literal) => acc.push(ReprPacked(literal)), @@ -777,20 +828,20 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { }; } if let Some(literal_error) = literal_error { - span_err!(diagnostic, item.span, E0589, + span_err!(diagnostic, item.span(), E0589, "invalid `repr(align)` attribute: {}", literal_error); } } else { if let Some(meta_item) = item.meta_item() { - if meta_item.name() == "align" { + if meta_item.check_name(sym::align) { if let MetaItemKind::NameValue(ref value) = meta_item.node { recognised = true; - let mut err = struct_span_err!(diagnostic, item.span, E0693, + let mut err = struct_span_err!(diagnostic, item.span(), E0693, "incorrect `repr(align)` attribute format"); match value.node { ast::LitKind::Int(int, ast::LitIntType::Unsuffixed) => { err.span_suggestion( - item.span, + item.span(), "use parentheses instead", format!("align({})", int), Applicability::MachineApplicable @@ -798,7 +849,7 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { } ast::LitKind::Str(s, _) => { err.span_suggestion( - item.span, + item.span(), "use parentheses instead", format!("align({})", s), Applicability::MachineApplicable @@ -813,7 +864,7 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { } if !recognised { // Not a word we recognize - span_err!(diagnostic, item.span, E0552, + span_err!(diagnostic, item.span(), E0552, "unrecognized representation hint"); } } @@ -822,22 +873,127 @@ pub fn find_repr_attrs(sess: &ParseSess, attr: &Attribute) -> Vec<ReprAttr> { acc } -fn int_type_of_word(s: &str) -> Option<IntType> { +fn int_type_of_word(s: Symbol) -> Option<IntType> { use IntType::*; match s { - "i8" => Some(SignedInt(ast::IntTy::I8)), - "u8" => Some(UnsignedInt(ast::UintTy::U8)), - "i16" => Some(SignedInt(ast::IntTy::I16)), - "u16" => Some(UnsignedInt(ast::UintTy::U16)), - "i32" => Some(SignedInt(ast::IntTy::I32)), - "u32" => Some(UnsignedInt(ast::UintTy::U32)), - "i64" => Some(SignedInt(ast::IntTy::I64)), - "u64" => Some(UnsignedInt(ast::UintTy::U64)), - "i128" => Some(SignedInt(ast::IntTy::I128)), - "u128" => Some(UnsignedInt(ast::UintTy::U128)), - "isize" => Some(SignedInt(ast::IntTy::Isize)), - "usize" => Some(UnsignedInt(ast::UintTy::Usize)), + sym::i8 => Some(SignedInt(ast::IntTy::I8)), + sym::u8 => Some(UnsignedInt(ast::UintTy::U8)), + sym::i16 => Some(SignedInt(ast::IntTy::I16)), + sym::u16 => Some(UnsignedInt(ast::UintTy::U16)), + sym::i32 => Some(SignedInt(ast::IntTy::I32)), + sym::u32 => Some(UnsignedInt(ast::UintTy::U32)), + sym::i64 => Some(SignedInt(ast::IntTy::I64)), + sym::u64 => Some(UnsignedInt(ast::UintTy::U64)), + sym::i128 => Some(SignedInt(ast::IntTy::I128)), + sym::u128 => Some(UnsignedInt(ast::UintTy::U128)), + sym::isize => Some(SignedInt(ast::IntTy::Isize)), + sym::usize => Some(UnsignedInt(ast::UintTy::Usize)), _ => None } } + +pub enum TransparencyError { + UnknownTransparency(Symbol, Span), + MultipleTransparencyAttrs(Span, Span), +} + +pub fn find_transparency( + attrs: &[Attribute], is_legacy: bool +) -> (Transparency, Option<TransparencyError>) { + let mut transparency = None; + let mut error = None; + for attr in attrs { + if attr.check_name(sym::rustc_macro_transparency) { + if let Some((_, old_span)) = transparency { + error = Some(TransparencyError::MultipleTransparencyAttrs(old_span, attr.span)); + break; + } else if let Some(value) = attr.value_str() { + transparency = Some((match &*value.as_str() { + "transparent" => Transparency::Transparent, + "semitransparent" => Transparency::SemiTransparent, + "opaque" => Transparency::Opaque, + _ => { + error = Some(TransparencyError::UnknownTransparency(value, attr.span)); + continue; + } + }, attr.span)); + } + } + } + let fallback = if is_legacy { Transparency::SemiTransparent } else { Transparency::Opaque }; + (transparency.map_or(fallback, |t| t.0), error) +} + +pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, name: Symbol) { + // All the built-in macro attributes are "words" at the moment. + let template = AttributeTemplate { word: true, list: None, name_value_str: None }; + let attr = ecx.attribute(meta_item.clone()); + check_builtin_attribute(ecx.parse_sess, &attr, name, template); +} + +crate fn check_builtin_attribute( + sess: &ParseSess, attr: &ast::Attribute, name: Symbol, template: AttributeTemplate +) { + // Some special attributes like `cfg` must be checked + // before the generic check, so we skip them here. + let should_skip = |name| name == sym::cfg; + // Some of previously accepted forms were used in practice, + // report them as warnings for now. + let should_warn = |name| name == sym::doc || name == sym::ignore || + name == sym::inline || name == sym::link || + name == sym::test || name == sym::bench; + + match attr.parse_meta(sess) { + Ok(meta) => if !should_skip(name) && !template.compatible(&meta.node) { + let error_msg = format!("malformed `{}` attribute input", name); + let mut msg = "attribute must be of the form ".to_owned(); + let mut suggestions = vec![]; + let mut first = true; + if template.word { + first = false; + let code = format!("#[{}]", name); + msg.push_str(&format!("`{}`", &code)); + suggestions.push(code); + } + if let Some(descr) = template.list { + if !first { + msg.push_str(" or "); + } + first = false; + let code = format!("#[{}({})]", name, descr); + msg.push_str(&format!("`{}`", &code)); + suggestions.push(code); + } + if let Some(descr) = template.name_value_str { + if !first { + msg.push_str(" or "); + } + let code = format!("#[{} = \"{}\"]", name, descr); + msg.push_str(&format!("`{}`", &code)); + suggestions.push(code); + } + if should_warn(name) { + sess.buffer_lint( + BufferedEarlyLintId::IllFormedAttributeInput, + meta.span, + ast::CRATE_NODE_ID, + &msg, + ); + } else { + sess.span_diagnostic.struct_span_err(meta.span, &error_msg) + .span_suggestions( + meta.span, + if suggestions.len() == 1 { + "must be of the form" + } else { + "the following are the possible correct uses" + }, + suggestions.into_iter(), + Applicability::HasPlaceholders, + ).emit(); + } + } + Err(mut err) => err.emit(), + } +} diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index b5fc8507314..a9d3227b3a8 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -2,27 +2,24 @@ mod builtin; -pub use builtin::{ - cfg_matches, contains_feature_attr, eval_condition, find_crate_name, find_deprecation, - find_repr_attrs, find_stability, find_unwind_attr, Deprecation, InlineAttr, OptimizeAttr, - IntType, ReprAttr, RustcDeprecation, Stability, StabilityLevel, UnwindAttr, -}; +pub use builtin::*; pub use IntType::*; pub use ReprAttr::*; pub use StabilityLevel::*; +pub use crate::ast::Attribute; use crate::ast; -use crate::ast::{AttrId, Attribute, AttrStyle, Name, Ident, Path, PathSegment}; -use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NestedMetaItemKind}; -use crate::ast::{Lit, LitKind, Expr, ExprKind, Item, Local, Stmt, StmtKind, GenericParam}; +use crate::ast::{AttrId, AttrStyle, Name, Ident, Path, PathSegment}; +use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem}; +use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam}; use crate::mut_visit::visit_clobber; -use crate::source_map::{BytePos, Spanned, respan, dummy_spanned}; +use crate::source_map::{BytePos, Spanned, dummy_spanned}; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::parser::Parser; use crate::parse::{self, ParseSess, PResult}; use crate::parse::token::{self, Token}; use crate::ptr::P; -use crate::symbol::Symbol; +use crate::symbol::{sym, Symbol}; use crate::ThinVec; use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; use crate::GLOBALS; @@ -34,7 +31,7 @@ use std::iter; use std::ops::DerefMut; pub fn mark_used(attr: &Attribute) { - debug!("Marking {:?} as used.", attr); + debug!("marking {:?} as used", attr); GLOBALS.with(|globals| { globals.used_attrs.lock().insert(attr.id); }); @@ -47,7 +44,7 @@ pub fn is_used(attr: &Attribute) -> bool { } pub fn mark_known(attr: &Attribute) { - debug!("Marking {:?} as known.", attr); + debug!("marking {:?} as known", attr); GLOBALS.with(|globals| { globals.known_attrs.lock().insert(attr.id); }); @@ -60,40 +57,37 @@ pub fn is_known(attr: &Attribute) -> bool { } pub fn is_known_lint_tool(m_item: Ident) -> bool { - ["clippy"].contains(&m_item.as_str().as_ref()) + [sym::clippy, sym::rustc].contains(&m_item.name) } impl NestedMetaItem { - /// Returns the MetaItem if self is a NestedMetaItemKind::MetaItem. + /// Returns the MetaItem if self is a NestedMetaItem::MetaItem. pub fn meta_item(&self) -> Option<&MetaItem> { - match self.node { - NestedMetaItemKind::MetaItem(ref item) => Some(item), + match *self { + NestedMetaItem::MetaItem(ref item) => Some(item), _ => None } } - /// Returns the Lit if self is a NestedMetaItemKind::Literal. + /// Returns the Lit if self is a NestedMetaItem::Literal. pub fn literal(&self) -> Option<&Lit> { - match self.node { - NestedMetaItemKind::Literal(ref lit) => Some(lit), + match *self { + NestedMetaItem::Literal(ref lit) => Some(lit), _ => None } } - /// Returns the Span for `self`. - pub fn span(&self) -> Span { - self.span - } - /// Returns `true` if this list item is a MetaItem with a name of `name`. - pub fn check_name(&self, name: &str) -> bool { + pub fn check_name(&self, name: Symbol) -> bool { self.meta_item().map_or(false, |meta_item| meta_item.check_name(name)) } - /// Returns the name of the meta item, e.g., `foo` in `#[foo]`, - /// `#[foo="bar"]` and `#[foo(bar)]`, if self is a MetaItem - pub fn name(&self) -> Option<Name> { - self.meta_item().and_then(|meta_item| Some(meta_item.name())) + /// For a single-segment meta-item returns its name, otherwise returns `None`. + pub fn ident(&self) -> Option<Ident> { + self.meta_item().and_then(|meta_item| meta_item.ident()) + } + pub fn name_or_empty(&self) -> Symbol { + self.ident().unwrap_or(Ident::invalid()).name } /// Gets the string value if self is a MetaItem and the MetaItem is a @@ -108,25 +102,14 @@ impl NestedMetaItem { |meta_item| meta_item.meta_item_list().and_then( |meta_item_list| { if meta_item_list.len() == 1 { - let nested_item = &meta_item_list[0]; - if nested_item.is_literal() { - Some((meta_item.name(), nested_item.literal().unwrap())) - } else { - None + if let Some(ident) = meta_item.ident() { + if let Some(lit) = meta_item_list[0].literal() { + return Some((ident.name, lit)); + } } } - else { - None - }})) - } - - /// Returns a MetaItem if self is a MetaItem with Kind Word. - pub fn word(&self) -> Option<&MetaItem> { - self.meta_item().and_then(|meta_item| if meta_item.is_word() { - Some(meta_item) - } else { - None - }) + None + })) } /// Gets a list of inner meta items from a list MetaItem type. @@ -146,7 +129,7 @@ impl NestedMetaItem { /// Returns `true` if self is a MetaItem and the meta item is a word. pub fn is_word(&self) -> bool { - self.word().is_some() + self.meta_item().map_or(false, |meta_item| meta_item.is_word()) } /// Returns `true` if self is a MetaItem and the meta item is a ValueString. @@ -160,16 +143,12 @@ impl NestedMetaItem { } } -fn name_from_path(path: &Path) -> Name { - path.segments.last().expect("empty path in attribute").ident.name -} - impl Attribute { /// Returns `true` if the attribute's path matches the argument. If it matches, then the /// attribute is marked as used. /// /// To check the attribute name without marking it used, use the `path` field directly. - pub fn check_name(&self, name: &str) -> bool { + pub fn check_name(&self, name: Symbol) -> bool { let matches = self.path == name; if matches { mark_used(self); @@ -177,10 +156,16 @@ impl Attribute { matches } - /// Returns the **last** segment of the name of this attribute. - /// e.g., `foo` for `#[foo]`, `skip` for `#[rustfmt::skip]`. - pub fn name(&self) -> Name { - name_from_path(&self.path) + /// For a single-segment attribute returns its name, otherwise returns `None`. + pub fn ident(&self) -> Option<Ident> { + if self.path.segments.len() == 1 { + Some(self.path.segments[0].ident) + } else { + None + } + } + pub fn name_or_empty(&self) -> Symbol { + self.ident().unwrap_or(Ident::invalid()).name } pub fn value_str(&self) -> Option<Symbol> { @@ -195,11 +180,7 @@ impl Attribute { } pub fn is_word(&self) -> bool { - self.path.segments.len() == 1 && self.tokens.is_empty() - } - - pub fn span(&self) -> Span { - self.span + self.tokens.is_empty() } pub fn is_meta_item_list(&self) -> bool { @@ -213,8 +194,16 @@ impl Attribute { } impl MetaItem { - pub fn name(&self) -> Name { - name_from_path(&self.ident) + /// For a single-segment meta-item returns its name, otherwise returns `None`. + pub fn ident(&self) -> Option<Ident> { + if self.path.segments.len() == 1 { + Some(self.path.segments[0].ident) + } else { + None + } + } + pub fn name_or_empty(&self) -> Symbol { + self.ident().unwrap_or(Ident::invalid()).name } // #[attribute(name = "value")] @@ -252,10 +241,8 @@ impl MetaItem { } } - pub fn span(&self) -> Span { self.span } - - pub fn check_name(&self, name: &str) -> bool { - self.name() == name + pub fn check_name(&self, name: Symbol) -> bool { + self.path == name } pub fn is_value_str(&self) -> bool { @@ -265,14 +252,6 @@ impl MetaItem { pub fn is_meta_item_list(&self) -> bool { self.meta_item_list().is_some() } - - pub fn is_scoped(&self) -> Option<Ident> { - if self.ident.segments.len() > 1 { - Some(self.ident.segments[0].ident) - } else { - None - } - } } impl Attribute { @@ -280,7 +259,7 @@ impl Attribute { pub fn meta(&self) -> Option<MetaItem> { let mut tokens = self.tokens.trees().peekable(); Some(MetaItem { - ident: self.path.clone(), + path: self.path.clone(), node: if let Some(node) = MetaItemKind::from_tokens(&mut tokens) { if tokens.peek().is_some() { return None; @@ -296,7 +275,14 @@ impl Attribute { pub fn parse<'a, T, F>(&self, sess: &'a ParseSess, mut f: F) -> PResult<'a, T> where F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, { - let mut parser = Parser::new(sess, self.tokens.clone(), None, false, false); + let mut parser = Parser::new( + sess, + self.tokens.clone(), + None, + false, + false, + Some("attribute"), + ); let result = f(&mut parser)?; if parser.token != token::Eof { parser.unexpected()?; @@ -326,7 +312,7 @@ impl Attribute { pub fn parse_meta<'a>(&self, sess: &'a ParseSess) -> PResult<'a, MetaItem> { Ok(MetaItem { - ident: self.path.clone(), + path: self.path.clone(), node: self.parse(sess, |parser| parser.parse_meta_item_kind())?, span: self.span, }) @@ -341,15 +327,16 @@ impl Attribute { if self.is_sugared_doc { let comment = self.value_str().unwrap(); let meta = mk_name_value_item_str( - Ident::from_str("doc"), + Ident::with_empty_ctxt(sym::doc), dummy_spanned(Symbol::intern(&strip_doc_comment_decoration(&comment.as_str())))); - let mut attr = if self.style == ast::AttrStyle::Outer { - mk_attr_outer(self.span, self.id, meta) - } else { - mk_attr_inner(self.span, self.id, meta) - }; - attr.is_sugared_doc = true; - f(&attr) + f(&Attribute { + id: self.id, + style: self.style, + path: meta.path, + tokens: meta.node.tokens(meta.span), + is_sugared_doc: true, + span: self.span, + }) } else { f(self) } @@ -359,27 +346,29 @@ impl Attribute { /* Constructors */ pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem { - let value = respan(value.span, LitKind::Str(value.node, ast::StrStyle::Cooked)); - mk_name_value_item(ident.span.to(value.span), ident, value) + let lit_kind = LitKind::Str(value.node, ast::StrStyle::Cooked); + mk_name_value_item(ident, lit_kind, value.span) } -pub fn mk_name_value_item(span: Span, ident: Ident, value: ast::Lit) -> MetaItem { - MetaItem { ident: Path::from_ident(ident), span, node: MetaItemKind::NameValue(value) } +pub fn mk_name_value_item(ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem { + let lit = Lit::from_lit_kind(lit_kind, lit_span); + let span = ident.span.to(lit_span); + MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(lit) } } -pub fn mk_list_item(span: Span, ident: Ident, items: Vec<NestedMetaItem>) -> MetaItem { - MetaItem { ident: Path::from_ident(ident), span, node: MetaItemKind::List(items) } +pub fn mk_list_item(ident: Ident, items: Vec<NestedMetaItem>) -> MetaItem { + MetaItem { path: Path::from_ident(ident), span: ident.span, node: MetaItemKind::List(items) } } pub fn mk_word_item(ident: Ident) -> MetaItem { - MetaItem { ident: Path::from_ident(ident), span: ident.span, node: MetaItemKind::Word } + MetaItem { path: Path::from_ident(ident), span: ident.span, node: MetaItemKind::Word } } pub fn mk_nested_word_item(ident: Ident) -> NestedMetaItem { - respan(ident.span, NestedMetaItemKind::MetaItem(mk_word_item(ident))) + NestedMetaItem::MetaItem(mk_word_item(ident)) } -pub fn mk_attr_id() -> AttrId { +crate fn mk_attr_id() -> AttrId { use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -390,75 +379,66 @@ pub fn mk_attr_id() -> AttrId { AttrId(id) } -/// Returns an inner attribute with the given value. -pub fn mk_attr_inner(span: Span, id: AttrId, item: MetaItem) -> Attribute { - mk_spanned_attr_inner(span, id, item) -} - /// Returns an inner attribute with the given value and span. -pub fn mk_spanned_attr_inner(sp: Span, id: AttrId, item: MetaItem) -> Attribute { +pub fn mk_attr_inner(item: MetaItem) -> Attribute { Attribute { - id, + id: mk_attr_id(), style: ast::AttrStyle::Inner, - path: item.ident, + path: item.path, tokens: item.node.tokens(item.span), is_sugared_doc: false, - span: sp, + span: item.span, } } -/// Returns an outer attribute with the given value. -pub fn mk_attr_outer(span: Span, id: AttrId, item: MetaItem) -> Attribute { - mk_spanned_attr_outer(span, id, item) -} - /// Returns an outer attribute with the given value and span. -pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute { +pub fn mk_attr_outer(item: MetaItem) -> Attribute { Attribute { - id, + id: mk_attr_id(), style: ast::AttrStyle::Outer, - path: item.ident, + path: item.path, tokens: item.node.tokens(item.span), is_sugared_doc: false, - span: sp, + span: item.span, } } -pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute { +pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute { let style = doc_comment_style(&text.as_str()); - let lit = respan(span, LitKind::Str(text, ast::StrStyle::Cooked)); + let lit_kind = LitKind::Str(text, ast::StrStyle::Cooked); + let lit = Lit::from_lit_kind(lit_kind, span); Attribute { - id, + id: mk_attr_id(), style, - path: Path::from_ident(Ident::from_str("doc").with_span_pos(span)), + path: Path::from_ident(Ident::with_empty_ctxt(sym::doc).with_span_pos(span)), tokens: MetaItemKind::NameValue(lit).tokens(span), is_sugared_doc: true, span, } } -pub fn list_contains_name(items: &[NestedMetaItem], name: &str) -> bool { +pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { items.iter().any(|item| { item.check_name(name) }) } -pub fn contains_name(attrs: &[Attribute], name: &str) -> bool { +pub fn contains_name(attrs: &[Attribute], name: Symbol) -> bool { attrs.iter().any(|item| { item.check_name(name) }) } -pub fn find_by_name<'a>(attrs: &'a [Attribute], name: &str) -> Option<&'a Attribute> { +pub fn find_by_name(attrs: &[Attribute], name: Symbol) -> Option<&Attribute> { attrs.iter().find(|attr| attr.check_name(name)) } -pub fn filter_by_name<'a>(attrs: &'a [Attribute], name: &'a str) - -> impl Iterator<Item = &'a Attribute> { +pub fn filter_by_name(attrs: &[Attribute], name: Symbol) + -> impl Iterator<Item=&Attribute> { attrs.iter().filter(move |attr| attr.check_name(name)) } -pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: &str) -> Option<Symbol> { +pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option<Symbol> { attrs.iter() .find(|at| at.check_name(name)) .and_then(|at| at.value_str()) @@ -468,16 +448,15 @@ impl MetaItem { fn tokens(&self) -> TokenStream { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); - for (i, segment) in self.ident.segments.iter().enumerate() { + for (i, segment) in self.path.segments.iter().enumerate() { let is_first = i == 0; if !is_first { let mod_sep_span = Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt()); - idents.push(TokenTree::Token(mod_sep_span, Token::ModSep).into()); + idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); } - idents.push(TokenTree::Token(segment.ident.span, - Token::from_ast_ident(segment.ident)).into()); + idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); @@ -488,27 +467,29 @@ impl MetaItem { where I: Iterator<Item = TokenTree>, { // FIXME: Share code with `parse_path`. - let ident = match tokens.next() { - Some(TokenTree::Token(span, token @ Token::Ident(..))) | - Some(TokenTree::Token(span, token @ Token::ModSep)) => 'arm: { - let mut segments = if let Token::Ident(ident, _) = token { - if let Some(TokenTree::Token(_, Token::ModSep)) = tokens.peek() { + let path = match tokens.next() { + Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) | + Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: { + let mut segments = if let token::Ident(name, _) = kind { + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) + = tokens.peek() { tokens.next(); - vec![PathSegment::from_ident(ident.with_span_pos(span))] + vec![PathSegment::from_ident(Ident::new(name, span))] } else { - break 'arm Path::from_ident(ident.with_span_pos(span)); + break 'arm Path::from_ident(Ident::new(name, span)); } } else { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(span, - Token::Ident(ident, _))) = tokens.next() { - segments.push(PathSegment::from_ident(ident.with_span_pos(span))); + if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) + = tokens.next() { + segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { return None; } - if let Some(TokenTree::Token(_, Token::ModSep)) = tokens.peek() { + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) + = tokens.peek() { tokens.next(); } else { break; @@ -517,7 +498,7 @@ impl MetaItem { let span = span.with_hi(segments.last().unwrap().ident.span.hi()); Path { span, segments } } - Some(TokenTree::Token(_, Token::Interpolated(nt))) => match *nt { + Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt { token::Nonterminal::NtIdent(ident, _) => Path::from_ident(ident), token::Nonterminal::NtMeta(ref meta) => return Some(meta.clone()), token::Nonterminal::NtPath(ref path) => path.clone(), @@ -529,11 +510,11 @@ impl MetaItem { let node = MetaItemKind::from_tokens(tokens)?; let hi = match node { MetaItemKind::NameValue(ref lit) => lit.span.hi(), - MetaItemKind::List(..) => list_closing_paren_pos.unwrap_or(ident.span.hi()), - _ => ident.span.hi(), + MetaItemKind::List(..) => list_closing_paren_pos.unwrap_or(path.span.hi()), + _ => path.span.hi(), }; - let span = ident.span.with_hi(hi); - Some(MetaItem { ident, node, span }) + let span = path.span.with_hi(hi); + Some(MetaItem { path, node, span }) } } @@ -542,7 +523,7 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::empty(), MetaItemKind::NameValue(ref lit) => { - let mut vec = vec![TokenTree::Token(span, Token::Eq).into()]; + let mut vec = vec![TokenTree::token(token::Eq, span).into()]; lit.tokens().append_to_tree_and_joint_vec(&mut vec); TokenStream::new(vec) } @@ -550,9 +531,9 @@ impl MetaItemKind { let mut tokens = Vec::new(); for (i, item) in list.iter().enumerate() { if i > 0 { - tokens.push(TokenTree::Token(span, Token::Comma).into()); + tokens.push(TokenTree::token(token::Comma, span).into()); } - item.node.tokens().append_to_tree_and_joint_vec(&mut tokens); + item.tokens().append_to_tree_and_joint_vec(&mut tokens); } TokenTree::Delimited( DelimSpan::from_single(span), @@ -567,11 +548,10 @@ impl MetaItemKind { where I: Iterator<Item = TokenTree>, { let delimited = match tokens.peek().cloned() { - Some(TokenTree::Token(_, token::Eq)) => { + Some(TokenTree::Token(token)) if token == token::Eq => { tokens.next(); - return if let Some(TokenTree::Token(span, token)) = tokens.next() { - LitKind::from_token(token) - .map(|lit| MetaItemKind::NameValue(Spanned { node: lit, span: span })) + return if let Some(TokenTree::Token(token)) = tokens.next() { + Lit::from_token(&token).ok().map(MetaItemKind::NameValue) } else { None }; @@ -586,10 +566,10 @@ impl MetaItemKind { let mut tokens = delimited.into_trees().peekable(); let mut result = Vec::new(); while let Some(..) = tokens.peek() { - let item = NestedMetaItemKind::from_tokens(&mut tokens)?; - result.push(respan(item.span(), item)); + let item = NestedMetaItem::from_tokens(&mut tokens)?; + result.push(item); match tokens.next() { - None | Some(TokenTree::Token(_, Token::Comma)) => {} + None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {} _ => return None, } } @@ -597,107 +577,32 @@ impl MetaItemKind { } } -impl NestedMetaItemKind { - fn span(&self) -> Span { +impl NestedMetaItem { + pub fn span(&self) -> Span { match *self { - NestedMetaItemKind::MetaItem(ref item) => item.span, - NestedMetaItemKind::Literal(ref lit) => lit.span, + NestedMetaItem::MetaItem(ref item) => item.span, + NestedMetaItem::Literal(ref lit) => lit.span, } } fn tokens(&self) -> TokenStream { match *self { - NestedMetaItemKind::MetaItem(ref item) => item.tokens(), - NestedMetaItemKind::Literal(ref lit) => lit.tokens(), + NestedMetaItem::MetaItem(ref item) => item.tokens(), + NestedMetaItem::Literal(ref lit) => lit.tokens(), } } - fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItemKind> + fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem> where I: Iterator<Item = TokenTree>, { - if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() { - if let Some(node) = LitKind::from_token(token) { + if let Some(TokenTree::Token(token)) = tokens.peek() { + if let Ok(lit) = Lit::from_token(token) { tokens.next(); - return Some(NestedMetaItemKind::Literal(respan(span, node))); - } - } - - MetaItem::from_tokens(tokens).map(NestedMetaItemKind::MetaItem) - } -} - -impl Lit { - crate fn tokens(&self) -> TokenStream { - TokenTree::Token(self.span, self.node.token()).into() - } -} - -impl LitKind { - fn token(&self) -> Token { - use std::ascii; - - match *self { - LitKind::Str(string, ast::StrStyle::Cooked) => { - let escaped = string.as_str().escape_default().to_string(); - Token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None) - } - LitKind::Str(string, ast::StrStyle::Raw(n)) => { - Token::Literal(token::Lit::StrRaw(string, n), None) - } - LitKind::ByteStr(ref bytes) => { - let string = bytes.iter().cloned().flat_map(ascii::escape_default) - .map(Into::<char>::into).collect::<String>(); - Token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None) - } - LitKind::Byte(byte) => { - let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect(); - Token::Literal(token::Lit::Byte(Symbol::intern(&string)), None) - } - LitKind::Char(ch) => { - let string: String = ch.escape_default().map(Into::<char>::into).collect(); - Token::Literal(token::Lit::Char(Symbol::intern(&string)), None) + return Some(NestedMetaItem::Literal(lit)); } - LitKind::Int(n, ty) => { - let suffix = match ty { - ast::LitIntType::Unsigned(ty) => Some(Symbol::intern(ty.ty_to_string())), - ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())), - ast::LitIntType::Unsuffixed => None, - }; - Token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix) - } - LitKind::Float(symbol, ty) => { - Token::Literal(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string()))) - } - LitKind::FloatUnsuffixed(symbol) => Token::Literal(token::Lit::Float(symbol), None), - LitKind::Bool(value) => Token::Ident(Ident::with_empty_ctxt(Symbol::intern(if value { - "true" - } else { - "false" - })), false), - LitKind::Err(val) => Token::Literal(token::Lit::Err(val), None), } - } - fn from_token(token: Token) -> Option<LitKind> { - match token { - Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)), - Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)), - Token::Interpolated(nt) => match *nt { - token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node { - ExprKind::Lit(ref lit) => Some(lit.node.clone()), - _ => None, - }, - _ => None, - }, - Token::Literal(lit, suf) => { - let (suffix_illegal, result) = parse::lit_token(lit, suf, None); - if suffix_illegal && suf.is_some() { - return None; - } - result - } - _ => None, - } + MetaItem::from_tokens(tokens).map(NestedMetaItem::MetaItem) } } @@ -807,7 +712,7 @@ macro_rules! derive_has_attrs { derive_has_attrs! { Item, Expr, Local, ast::ForeignItem, ast::StructField, ast::ImplItem, ast::TraitItem, ast::Arm, - ast::Field, ast::FieldPat, ast::Variant_ + ast::Field, ast::FieldPat, ast::Variant_, ast::Arg } pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -> ast::Crate { @@ -818,9 +723,9 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - raw_attr.clone(), ); - let start_span = parser.span; + let start_span = parser.token.span; let (path, tokens) = panictry!(parser.parse_meta_item_unrestricted()); - let end_span = parser.span; + let end_span = parser.token.span; if parser.token != token::Eof { parse_sess.span_diagnostic .span_err(start_span.to(end_span), "invalid crate attribute"); diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index 4e4432a3f33..1ab367f73c1 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -12,6 +12,7 @@ use crate::edition::Edition; use crate::mut_visit::*; use crate::parse::{token, ParseSess}; use crate::ptr::P; +use crate::symbol::sym; use crate::util::map_in_place::MapInPlace; use errors::Applicability; @@ -24,8 +25,8 @@ pub struct StripUnconfigured<'a> { } // `cfg_attr`-process the crate's attributes and compute the crate's features. -pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition) - -> (ast::Crate, Features) { +pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, + allow_features: &Option<Vec<String>>) -> (ast::Crate, Features) { let features; { let mut strip_unconfigured = StripUnconfigured { @@ -43,7 +44,7 @@ pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition) return (krate, Features::new()); } - features = get_features(&sess.span_diagnostic, &krate.attrs, edition); + features = get_features(&sess.span_diagnostic, &krate.attrs, edition, allow_features); // Avoid reconfiguring malformed `cfg_attr`s if err_count == sess.span_diagnostic.err_count() { @@ -90,9 +91,25 @@ impl<'a> StripUnconfigured<'a> { /// is in the original source file. Gives a compiler error if the syntax of /// the attribute is incorrect. fn process_cfg_attr(&mut self, attr: ast::Attribute) -> Vec<ast::Attribute> { - if !attr.check_name("cfg_attr") { + if attr.path != sym::cfg_attr { return vec![attr]; } + if attr.tokens.is_empty() { + self.sess.span_diagnostic + .struct_span_err( + attr.span, + "malformed `cfg_attr` attribute input", + ).span_suggestion( + attr.span, + "missing condition and attribute", + "#[cfg_attr(condition, attribute, other_attribute, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).note("for more information, visit \ + <https://doc.rust-lang.org/reference/conditional-compilation.html\ + #the-cfg_attr-attribute>") + .emit(); + return vec![]; + } let (cfg_predicate, expanded_attrs) = match attr.parse(self.sess, |parser| { parser.expect(&token::OpenDelim(token::Paren))?; @@ -104,7 +121,7 @@ impl<'a> StripUnconfigured<'a> { let mut expanded_attrs = Vec::with_capacity(1); while !parser.check(&token::CloseDelim(token::Paren)) { - let lo = parser.span.lo(); + let lo = parser.token.span.lo(); let (path, tokens) = parser.parse_meta_item_unrestricted()?; expanded_attrs.push((path, tokens, parser.prev_span.with_lo(lo))); parser.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Paren)])?; @@ -116,17 +133,18 @@ impl<'a> StripUnconfigured<'a> { Ok(result) => result, Err(mut e) => { e.emit(); - return Vec::new(); + return vec![]; } }; - // Check feature gate and lint on zero attributes in source. Even if the feature is gated, - // we still compute as if it wasn't, since the emitted error will stop compilation further - // along the compilation. - if expanded_attrs.len() == 0 { - // FIXME: Emit unused attribute lint here. + // Lint on zero attributes in source. + if expanded_attrs.is_empty() { + return vec![attr]; } + // At this point we know the attribute is considered used. + attr::mark_used(&attr); + if attr::cfg_matches(&cfg_predicate, self.sess, self.features) { // We call `process_cfg_attr` recursively in case there's a // `cfg_attr` inside of another `cfg_attr`. E.g. @@ -142,7 +160,7 @@ impl<'a> StripUnconfigured<'a> { })) .collect() } else { - Vec::new() + vec![] } } @@ -181,13 +199,13 @@ impl<'a> StripUnconfigured<'a> { if nested_meta_items.is_empty() { return error(meta_item.span, "`cfg` predicate is not specified", ""); } else if nested_meta_items.len() > 1 { - return error(nested_meta_items.last().unwrap().span, + return error(nested_meta_items.last().unwrap().span(), "multiple `cfg` predicates are specified", ""); } match nested_meta_items[0].meta_item() { Some(meta_item) => attr::cfg_matches(meta_item, self.sess, self.features), - None => error(nested_meta_items[0].span, + None => error(nested_meta_items[0].span(), "`cfg` predicate key cannot be a literal", ""), } }) @@ -205,7 +223,7 @@ impl<'a> StripUnconfigured<'a> { pub fn maybe_emit_expr_attr_err(&self, attr: &ast::Attribute) { if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) { let mut err = feature_err(self.sess, - "stmt_expr_attributes", + sym::stmt_expr_attributes, attr.span, GateIssue::Language, EXPLAIN_STMT_ATTR_SYNTAX); @@ -223,12 +241,15 @@ impl<'a> StripUnconfigured<'a> { items.flat_map_in_place(|item| self.configure(item)); } + pub fn configure_generic_params(&mut self, params: &mut Vec<ast::GenericParam>) { + params.flat_map_in_place(|param| self.configure(param)); + } + fn configure_variant_data(&mut self, vdata: &mut ast::VariantData) { match vdata { - ast::VariantData::Struct(fields, _id) | - ast::VariantData::Tuple(fields, _id) => + ast::VariantData::Struct(fields, ..) | ast::VariantData::Tuple(fields, _) => fields.flat_map_in_place(|field| self.configure(field)), - ast::VariantData::Unit(_id) => {} + ast::VariantData::Unit(_) => {} } } @@ -282,20 +303,8 @@ impl<'a> StripUnconfigured<'a> { } } - /// Denies `#[cfg]` on generic parameters until we decide what to do with it. - /// See issue #51279. - pub fn disallow_cfg_on_generic_param(&mut self, param: &ast::GenericParam) { - for attr in param.attrs() { - let offending_attr = if attr.check_name("cfg") { - "cfg" - } else if attr.check_name("cfg_attr") { - "cfg_attr" - } else { - continue; - }; - let msg = format!("#[{}] cannot be applied on a generic parameter", offending_attr); - self.sess.span_diagnostic.span_err(attr.span, &msg); - } + pub fn configure_fn_decl(&mut self, fn_decl: &mut ast::FnDecl) { + fn_decl.inputs.flat_map_in_place(|arg| self.configure(arg)); } } @@ -348,8 +357,13 @@ impl<'a> MutVisitor for StripUnconfigured<'a> { self.configure_pat(pat); noop_visit_pat(pat, self) } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) { + self.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } } fn is_cfg(attr: &ast::Attribute) -> bool { - attr.check_name("cfg") + attr.check_name(sym::cfg) } diff --git a/src/libsyntax/diagnostics/macros.rs b/src/libsyntax/diagnostics/macros.rs index 6f7493ad597..b754d083376 100644 --- a/src/libsyntax/diagnostics/macros.rs +++ b/src/libsyntax/diagnostics/macros.rs @@ -170,19 +170,19 @@ macro_rules! help { #[macro_export] macro_rules! register_diagnostics { ($($code:tt),*) => ( - $(register_diagnostic! { $code })* + $($crate::register_diagnostic! { $code })* ); ($($code:tt),*,) => ( - $(register_diagnostic! { $code })* + $($crate::register_diagnostic! { $code })* ) } #[macro_export] macro_rules! register_long_diagnostics { ($($code:tt: $description:tt),*) => ( - $(register_diagnostic! { $code, $description })* + $($crate::register_diagnostic! { $code, $description })* ); ($($code:tt: $description:tt),*,) => ( - $(register_diagnostic! { $code, $description })* + $($crate::register_diagnostic! { $code, $description })* ) } diff --git a/src/libsyntax/diagnostics/metadata.rs b/src/libsyntax/diagnostics/metadata.rs index 704135fe1d5..53f37bb10bd 100644 --- a/src/libsyntax/diagnostics/metadata.rs +++ b/src/libsyntax/diagnostics/metadata.rs @@ -36,9 +36,9 @@ pub struct ErrorLocation { impl ErrorLocation { /// Creates an error location from a span. pub fn from_span(ecx: &ExtCtxt<'_>, sp: Span) -> ErrorLocation { - let loc = ecx.source_map().lookup_char_pos_adj(sp.lo()); + let loc = ecx.source_map().lookup_char_pos(sp.lo()); ErrorLocation { - filename: loc.filename, + filename: loc.file.name.clone(), line: loc.line } } diff --git a/src/libsyntax/diagnostics/plugin.rs b/src/libsyntax/diagnostics/plugin.rs index 21024eb41ef..80591ad304d 100644 --- a/src/libsyntax/diagnostics/plugin.rs +++ b/src/libsyntax/diagnostics/plugin.rs @@ -4,10 +4,9 @@ use std::env; use crate::ast::{self, Ident, Name}; use crate::source_map; use crate::ext::base::{ExtCtxt, MacEager, MacResult}; -use crate::ext::build::AstBuilder; -use crate::parse::token; +use crate::parse::token::{self, Token}; use crate::ptr::P; -use crate::symbol::{keywords, Symbol}; +use crate::symbol::kw; use crate::tokenstream::{TokenTree}; use smallvec::smallvec; @@ -33,13 +32,15 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>, span: Span, token_tree: &[TokenTree]) -> Box<dyn MacResult+'cx> { - let code = match (token_tree.len(), token_tree.get(0)) { - (1, Some(&TokenTree::Token(_, token::Ident(code, _)))) => code, + let code = match token_tree { + [ + TokenTree::Token(Token { kind: token::Ident(code, _), .. }) + ] => code, _ => unreachable!() }; ecx.parse_sess.registered_diagnostics.with_lock(|diagnostics| { - match diagnostics.get_mut(&code.name) { + match diagnostics.get_mut(&code) { // Previously used errors. Some(&mut ErrorInfo { description: _, use_site: Some(previous_span) }) => { ecx.struct_span_warn(span, &format!( @@ -66,20 +67,19 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, span: Span, token_tree: &[TokenTree]) -> Box<dyn MacResult+'cx> { - let (code, description) = match ( - token_tree.len(), - token_tree.get(0), - token_tree.get(1), - token_tree.get(2) - ) { - (1, Some(&TokenTree::Token(_, token::Ident(ref code, _))), None, None) => { - (code, None) + let (code, description) = match token_tree { + [ + TokenTree::Token(Token { kind: token::Ident(code, _), .. }) + ] => { + (*code, None) + }, + [ + TokenTree::Token(Token { kind: token::Ident(code, _), .. }), + TokenTree::Token(Token { kind: token::Comma, .. }), + TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), ..}) + ] => { + (*code, Some(*symbol)) }, - (3, Some(&TokenTree::Token(_, token::Ident(ref code, _))), - Some(&TokenTree::Token(_, token::Comma)), - Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => { - (code, Some(description)) - } _ => unreachable!() }; @@ -112,41 +112,28 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>, description, use_site: None }; - if diagnostics.insert(code.name, info).is_some() { + if diagnostics.insert(code, info).is_some() { ecx.span_err(span, &format!( "diagnostic code {} already registered", code )); } }); - let span = span.apply_mark(ecx.current_expansion.mark); - - let sym = Ident::new(Symbol::gensym(&format!("__register_diagnostic_{}", code)), span); - - MacEager::items(smallvec![ - ecx.item_mod( - span, - span, - sym, - vec![], - vec![], - ) - ]) + MacEager::items(smallvec![]) } -#[allow(deprecated)] pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, span: Span, token_tree: &[TokenTree]) -> Box<dyn MacResult+'cx> { assert_eq!(token_tree.len(), 3); - let (crate_name, name) = match (&token_tree[0], &token_tree[2]) { + let (crate_name, ident) = match (&token_tree[0], &token_tree[2]) { ( // Crate name. - &TokenTree::Token(_, token::Ident(ref crate_name, _)), + &TokenTree::Token(Token { kind: token::Ident(crate_name, _), .. }), // DIAGNOSTICS ident. - &TokenTree::Token(_, token::Ident(ref name, _)) - ) => (*&crate_name, name), + &TokenTree::Token(Token { kind: token::Ident(name, _), span }) + ) => (crate_name, Ident::new(name, span)), _ => unreachable!() }; @@ -160,7 +147,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, ecx.span_bug(span, &format!( "error writing metadata for triple `{}` and crate `{}`, error: {}, \ cause: {:?}", - target_triple, crate_name, e.description(), e.cause() + target_triple, crate_name, e.description(), e.source() )); } }); @@ -185,7 +172,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, (descriptions.len(), ecx.expr_vec(span, descriptions)) }); - let static_ = ecx.lifetime(span, keywords::StaticLifetime.ident()); + let static_ = ecx.lifetime(span, Ident::with_empty_ctxt(kw::StaticLifetime)); let ty_str = ecx.ty_rptr( span, ecx.ty_ident(span, ecx.ident_of("str")), @@ -209,7 +196,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>, MacEager::items(smallvec![ P(ast::Item { - ident: *name, + ident, attrs: Vec::new(), id: ast::DUMMY_NODE_ID, node: ast::ItemKind::Const( diff --git a/src/libsyntax/early_buffered_lints.rs b/src/libsyntax/early_buffered_lints.rs index 29cb9cd7f30..36c1da29299 100644 --- a/src/libsyntax/early_buffered_lints.rs +++ b/src/libsyntax/early_buffered_lints.rs @@ -3,17 +3,14 @@ //! Since we cannot have a dependency on `librustc`, we implement some types here that are somewhat //! redundant. Later, these types can be converted to types for use by the rest of the compiler. -use crate::syntax::ast::NodeId; +use crate::ast::NodeId; use syntax_pos::MultiSpan; /// Since we cannot import `LintId`s from `rustc::lint`, we define some Ids here which can later be /// passed to `rustc::lint::Lint::from_parser_lint_id` to get a `rustc::lint::Lint`. pub enum BufferedEarlyLintId { - /// Usage of `?` as a macro separator is deprecated. - QuestionMarkMacroSep, IllFormedAttributeInput, - /// Usage of a duplicate macro matcher binding name. - DuplicateMacroMatcherBindingName, + MetaVariableMisuse, } /// Stores buffered lint info which can later be passed to `librustc`. diff --git a/src/libsyntax/entry.rs b/src/libsyntax/entry.rs index 09e26e29d86..0b6cf30bd27 100644 --- a/src/libsyntax/entry.rs +++ b/src/libsyntax/entry.rs @@ -1,5 +1,6 @@ use crate::attr; use crate::ast::{Item, ItemKind}; +use crate::symbol::sym; pub enum EntryPointType { None, @@ -14,11 +15,11 @@ pub enum EntryPointType { pub fn entry_point_type(item: &Item, depth: usize) -> EntryPointType { match item.node { ItemKind::Fn(..) => { - if attr::contains_name(&item.attrs, "start") { + if attr::contains_name(&item.attrs, sym::start) { EntryPointType::Start - } else if attr::contains_name(&item.attrs, "main") { + } else if attr::contains_name(&item.attrs, sym::main) { EntryPointType::MainAttr - } else if item.ident.name == "main" { + } else if item.ident.name == sym::main { if depth == 1 { // This is a top-level function so can be 'main' EntryPointType::MainNamed diff --git a/src/libsyntax/diagnostic_list.rs b/src/libsyntax/error_codes.rs index 2c367f1f402..1ba29011f75 100644 --- a/src/libsyntax/diagnostic_list.rs +++ b/src/libsyntax/error_codes.rs @@ -1,5 +1,3 @@ -#![allow(non_snake_case)] - // Error messages for EXXXX errors. // Each message should start and end with a new line, and be wrapped to 80 characters. // In vim you can `:set tw=80` and use `gq` to wrap paragraphs. Use `:set tw=0` to disable. @@ -182,7 +180,7 @@ beta compilers will not comply. Example of erroneous code (on a stable compiler): ```ignore (depends on release channel) -#![feature(non_ascii_idents)] // error: #![feature] may not be used on the +#![feature(non_ascii_idents)] // error: `#![feature]` may not be used on the // stable release channel ``` @@ -363,6 +361,35 @@ and likely to change in the future. "##, +E0704: r##" +This error indicates that a incorrect visibility restriction was specified. + +Example of erroneous code: + +```compile_fail,E0704 +mod foo { + pub(foo) struct Bar { + x: i32 + } +} +``` + +To make struct `Bar` only visible in module `foo` the `in` keyword should be +used: +``` +mod foo { + pub(in crate::foo) struct Bar { + x: i32 + } +} +# fn main() {} +``` + +For more information see the Rust Reference on [Visibility]. + +[Visibility]: https://doc.rust-lang.org/reference/visibility-and-privacy.html +"##, + E0705: r##" A `#![feature]` attribute was declared for a feature that is stable in the current edition, but not in all editions. @@ -378,6 +405,21 @@ Erroneous code example: "##, +E0725: r##" +A feature attribute named a feature that was disallowed in the compiler +command line flags. + +Erroneous code example: + +```ignore (can't specify compiler flags from doctests) +#![feature(never_type)] // error: the feature `never_type` is not in + // the list of allowed features +``` + +Delete the offending feature attribute, or add it to the list of allowed +features in the `-Z allow_features` flag. +"##, + } register_diagnostics! { @@ -402,6 +444,5 @@ register_diagnostics! { E0693, // incorrect `repr(align)` attribute format E0694, // an unknown tool name found in scoped attributes E0703, // invalid ABI - E0704, // incorrect visibility restriction E0717, // rustc_promotable without stability attribute } diff --git a/src/libsyntax/ext/allocator.rs b/src/libsyntax/ext/allocator.rs new file mode 100644 index 00000000000..99aeb5414c5 --- /dev/null +++ b/src/libsyntax/ext/allocator.rs @@ -0,0 +1,75 @@ +use crate::{ast, attr, visit}; +use crate::symbol::{sym, Symbol}; +use syntax_pos::Span; + +#[derive(Clone, Copy)] +pub enum AllocatorKind { + Global, + DefaultLib, + DefaultExe, +} + +impl AllocatorKind { + pub fn fn_name(&self, base: &str) -> String { + match *self { + AllocatorKind::Global => format!("__rg_{}", base), + AllocatorKind::DefaultLib => format!("__rdl_{}", base), + AllocatorKind::DefaultExe => format!("__rde_{}", base), + } + } +} + +pub enum AllocatorTy { + Layout, + Ptr, + ResultPtr, + Unit, + Usize, +} + +pub struct AllocatorMethod { + pub name: &'static str, + pub inputs: &'static [AllocatorTy], + pub output: AllocatorTy, +} + +pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ + AllocatorMethod { + name: "alloc", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "dealloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout], + output: AllocatorTy::Unit, + }, + AllocatorMethod { + name: "realloc", + inputs: &[AllocatorTy::Ptr, AllocatorTy::Layout, AllocatorTy::Usize], + output: AllocatorTy::ResultPtr, + }, + AllocatorMethod { + name: "alloc_zeroed", + inputs: &[AllocatorTy::Layout], + output: AllocatorTy::ResultPtr, + }, +]; + +pub fn global_allocator_spans(krate: &ast::Crate) -> Vec<Span> { + struct Finder { name: Symbol, spans: Vec<Span> } + impl<'ast> visit::Visitor<'ast> for Finder { + fn visit_item(&mut self, item: &'ast ast::Item) { + if item.ident.name == self.name && + attr::contains_name(&item.attrs, sym::rustc_std_internal_symbol) { + self.spans.push(item.span); + } + visit::walk_item(self, item) + } + } + + let name = Symbol::intern(&AllocatorKind::Global.fn_name("alloc")); + let mut f = Finder { name, spans: Vec::new() }; + visit::walk_crate(&mut f, krate); + f.spans +} diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs index 452cc2f2c65..d69822c7c7f 100644 --- a/src/libsyntax/ext/base.rs +++ b/src/libsyntax/ext/base.rs @@ -1,22 +1,21 @@ -pub use SyntaxExtension::*; - -use crate::ast::{self, Attribute, Name, PatKind, MetaItem}; -use crate::attr::HasAttrs; +use crate::ast::{self, Attribute, Name, PatKind}; +use crate::attr::{HasAttrs, Stability, Deprecation}; use crate::source_map::{SourceMap, Spanned, respan}; use crate::edition::Edition; use crate::ext::expand::{self, AstFragment, Invocation}; -use crate::ext::hygiene::{self, Mark, SyntaxContext, Transparency}; +use crate::ext::hygiene::{ExpnId, SyntaxContext, Transparency}; use crate::mut_visit::{self, MutVisitor}; use crate::parse::{self, parser, DirectoryOwnership}; use crate::parse::token; use crate::ptr::P; -use crate::symbol::{keywords, Ident, Symbol}; -use crate::ThinVec; -use crate::tokenstream::{self, TokenStream}; +use crate::symbol::{kw, sym, Ident, Symbol}; +use crate::{ThinVec, MACRO_ARGUMENTS}; +use crate::tokenstream::{self, TokenStream, TokenTree}; use errors::{DiagnosticBuilder, DiagnosticId}; use smallvec::{smallvec, SmallVec}; -use syntax_pos::{Span, MultiSpan, DUMMY_SP}; +use syntax_pos::{FileName, Span, MultiSpan, DUMMY_SP}; +use syntax_pos::hygiene::{ExpnInfo, ExpnKind}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::{self, Lrc}; @@ -25,6 +24,7 @@ use std::path::PathBuf; use std::rc::Rc; use std::default::Default; +pub use syntax_pos::hygiene::MacroKind; #[derive(Debug,Clone)] pub enum Annotatable { @@ -137,29 +137,6 @@ impl Annotatable { } } -// A more flexible ItemDecorator. -pub trait MultiItemDecorator { - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - sp: Span, - meta_item: &ast::MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable)); -} - -impl<F> MultiItemDecorator for F - where F : Fn(&mut ExtCtxt<'_>, Span, &ast::MetaItem, &Annotatable, &mut dyn FnMut(Annotatable)) -{ - fn expand(&self, - ecx: &mut ExtCtxt<'_>, - sp: Span, - meta_item: &ast::MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable)) { - (*self)(ecx, sp, meta_item, item, push) - } -} - // `meta_item` is the annotation, and `item` is the item being modified. // FIXME Decorators should follow the same pattern too. pub trait MultiItemModifier { @@ -242,7 +219,6 @@ pub trait TTMacroExpander { ecx: &'cx mut ExtCtxt<'_>, span: Span, input: TokenStream, - def_span: Option<Span>, ) -> Box<dyn MacResult+'cx>; } @@ -259,16 +235,18 @@ impl<F> TTMacroExpander for F ecx: &'cx mut ExtCtxt<'_>, span: Span, input: TokenStream, - _def_span: Option<Span>, ) -> Box<dyn MacResult+'cx> { struct AvoidInterpolatedIdents; impl MutVisitor for AvoidInterpolatedIdents { fn visit_tt(&mut self, tt: &mut tokenstream::TokenTree) { - if let tokenstream::TokenTree::Token(_, token::Interpolated(nt)) = tt { - if let token::NtIdent(ident, is_raw) = **nt { - *tt = tokenstream::TokenTree::Token(ident.span, - token::Ident(ident, is_raw)); + if let tokenstream::TokenTree::Token(token) = tt { + if let token::Interpolated(nt) = &token.kind { + if let token::NtIdent(ident, is_raw) = **nt { + *tt = tokenstream::TokenTree::token( + token::Ident(ident.name, is_raw), ident.span + ); + } } } mut_visit::noop_visit_tt(tt, self) @@ -285,34 +263,6 @@ impl<F> TTMacroExpander for F } } -pub trait IdentMacroExpander { - fn expand<'cx>(&self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - ident: ast::Ident, - token_tree: Vec<tokenstream::TokenTree>) - -> Box<dyn MacResult+'cx>; -} - -pub type IdentMacroExpanderFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, ast::Ident, Vec<tokenstream::TokenTree>) - -> Box<dyn MacResult+'cx>; - -impl<F> IdentMacroExpander for F - where F : for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, ast::Ident, - Vec<tokenstream::TokenTree>) -> Box<dyn MacResult+'cx> -{ - fn expand<'cx>(&self, - cx: &'cx mut ExtCtxt<'_>, - sp: Span, - ident: ast::Ident, - token_tree: Vec<tokenstream::TokenTree>) - -> Box<dyn MacResult+'cx> - { - (*self)(cx, sp, ident, token_tree) - } -} - // Use a macro because forwarding to a simple function has type system issues macro_rules! make_stmts_default { ($me:expr) => { @@ -567,225 +517,204 @@ impl MacResult for DummyResult { } } -pub type BuiltinDeriveFn = - for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, &MetaItem, &Annotatable, &mut dyn FnMut(Annotatable)); - -/// Represents different kinds of macro invocations that can be resolved. -#[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] -pub enum MacroKind { - /// A bang macro - foo!() - Bang, - /// An attribute macro - #[foo] - Attr, - /// A derive attribute macro - #[derive(Foo)] - Derive, - /// A view of a procedural macro from the same crate that defines it. - ProcMacroStub, +/// A syntax extension kind. +pub enum SyntaxExtensionKind { + /// A token-based function-like macro. + Bang( + /// An expander with signature TokenStream -> TokenStream. + Box<dyn ProcMacro + sync::Sync + sync::Send>, + ), + + /// An AST-based function-like macro. + LegacyBang( + /// An expander with signature TokenStream -> AST. + Box<dyn TTMacroExpander + sync::Sync + sync::Send>, + ), + + /// A token-based attribute macro. + Attr( + /// An expander with signature (TokenStream, TokenStream) -> TokenStream. + /// The first TokenSteam is the attribute itself, the second is the annotated item. + /// The produced TokenSteam replaces the input TokenSteam. + Box<dyn AttrProcMacro + sync::Sync + sync::Send>, + ), + + /// An AST-based attribute macro. + LegacyAttr( + /// An expander with signature (AST, AST) -> AST. + /// The first AST fragment is the attribute itself, the second is the annotated item. + /// The produced AST fragment replaces the input AST fragment. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), + + /// A trivial attribute "macro" that does nothing, + /// only keeps the attribute and marks it as inert, + /// thus making it ineligible for further expansion. + NonMacroAttr { + /// Suppresses the `unused_attributes` lint for this attribute. + mark_used: bool, + }, + + /// A token-based derive macro. + Derive( + /// An expander with signature TokenStream -> TokenStream (not yet). + /// The produced TokenSteam is appended to the input TokenSteam. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), + + /// An AST-based derive macro. + LegacyDerive( + /// An expander with signature AST -> AST. + /// The produced AST fragment is appended to the input AST fragment. + Box<dyn MultiItemModifier + sync::Sync + sync::Send>, + ), } -impl MacroKind { - pub fn descr(self) -> &'static str { - match self { - MacroKind::Bang => "macro", - MacroKind::Attr => "attribute macro", - MacroKind::Derive => "derive macro", - MacroKind::ProcMacroStub => "crate-local procedural macro", - } - } +/// A struct representing a macro definition in "lowered" form ready for expansion. +pub struct SyntaxExtension { + /// A syntax extension kind. + pub kind: SyntaxExtensionKind, + /// Span of the macro definition. + pub span: Span, + /// Hygienic properties of spans produced by this macro by default. + pub default_transparency: Transparency, + /// Whitelist of unstable features that are treated as stable inside this macro. + pub allow_internal_unstable: Option<Lrc<[Symbol]>>, + /// Suppresses the `unsafe_code` lint for code produced by this macro. + pub allow_internal_unsafe: bool, + /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) for this macro. + pub local_inner_macros: bool, + /// The macro's stability info. + pub stability: Option<Stability>, + /// The macro's deprecation info. + pub deprecation: Option<Deprecation>, + /// Names of helper attributes registered by this macro. + pub helper_attrs: Vec<Symbol>, + /// Edition of the crate in which this macro is defined. + pub edition: Edition, + /// Built-in macros have a couple of special properties (meaning of `$crate`, + /// availability in `#[no_implicit_prelude]` modules), so we have to keep this flag. + pub is_builtin: bool, + /// We have to identify macros providing a `Copy` impl early for compatibility reasons. + pub is_derive_copy: bool, +} - pub fn article(self) -> &'static str { +impl SyntaxExtensionKind { + /// When a syntax extension is constructed, + /// its transparency can often be inferred from its kind. + fn default_transparency(&self) -> Transparency { match self { - MacroKind::Attr => "an", - _ => "a", + SyntaxExtensionKind::Bang(..) | + SyntaxExtensionKind::Attr(..) | + SyntaxExtensionKind::Derive(..) | + SyntaxExtensionKind::NonMacroAttr { .. } => Transparency::Opaque, + SyntaxExtensionKind::LegacyBang(..) | + SyntaxExtensionKind::LegacyAttr(..) | + SyntaxExtensionKind::LegacyDerive(..) => Transparency::SemiTransparent, } } } -/// An enum representing the different kinds of syntax extensions. -pub enum SyntaxExtension { - /// A trivial "extension" that does nothing, only keeps the attribute and marks it as known. - NonMacroAttr { mark_used: bool }, - - /// A syntax extension that is attached to an item and creates new items - /// based upon it. - /// - /// `#[derive(...)]` is a `MultiItemDecorator`. - /// - /// Prefer ProcMacro or MultiModifier since they are more flexible. - MultiDecorator(Box<dyn MultiItemDecorator + sync::Sync + sync::Send>), - - /// A syntax extension that is attached to an item and modifies it - /// in-place. Also allows decoration, i.e., creating new items. - MultiModifier(Box<dyn MultiItemModifier + sync::Sync + sync::Send>), - - /// A function-like procedural macro. TokenStream -> TokenStream. - ProcMacro { - expander: Box<dyn ProcMacro + sync::Sync + sync::Send>, - /// Whitelist of unstable features that are treated as stable inside this macro - allow_internal_unstable: Option<Lrc<[Symbol]>>, - edition: Edition, - }, - - /// An attribute-like procedural macro. TokenStream, TokenStream -> TokenStream. - /// The first TokenSteam is the attribute, the second is the annotated item. - /// Allows modification of the input items and adding new items, similar to - /// MultiModifier, but uses TokenStreams, rather than AST nodes. - AttrProcMacro(Box<dyn AttrProcMacro + sync::Sync + sync::Send>, Edition), - - /// A normal, function-like syntax extension. - /// - /// `bytes!` is a `NormalTT`. - NormalTT { - expander: Box<dyn TTMacroExpander + sync::Sync + sync::Send>, - def_info: Option<(ast::NodeId, Span)>, - /// Whether the contents of the macro can - /// directly use `#[unstable]` things. - /// - /// Only allows things that require a feature gate in the given whitelist - allow_internal_unstable: Option<Lrc<[Symbol]>>, - /// Whether the contents of the macro can use `unsafe` - /// without triggering the `unsafe_code` lint. - allow_internal_unsafe: bool, - /// Enables the macro helper hack (`ident!(...)` -> `$crate::ident!(...)`) - /// for a given macro. - local_inner_macros: bool, - /// The macro's feature name if it is unstable, and the stability feature - unstable_feature: Option<(Symbol, u32)>, - /// Edition of the crate in which the macro is defined - edition: Edition, - }, - - /// A function-like syntax extension that has an extra ident before - /// the block. - IdentTT { - expander: Box<dyn IdentMacroExpander + sync::Sync + sync::Send>, - span: Option<Span>, - allow_internal_unstable: Option<Lrc<[Symbol]>>, - }, - - /// An attribute-like procedural macro. TokenStream -> TokenStream. - /// The input is the annotated item. - /// Allows generating code to implement a Trait for a given struct - /// or enum item. - ProcMacroDerive(Box<dyn MultiItemModifier + sync::Sync + sync::Send>, - Vec<Symbol> /* inert attribute names */, Edition), - - /// An attribute-like procedural macro that derives a builtin trait. - BuiltinDerive(BuiltinDeriveFn), +impl SyntaxExtension { + /// Returns which kind of macro calls this syntax extension. + pub fn macro_kind(&self) -> MacroKind { + match self.kind { + SyntaxExtensionKind::Bang(..) | + SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang, + SyntaxExtensionKind::Attr(..) | + SyntaxExtensionKind::LegacyAttr(..) | + SyntaxExtensionKind::NonMacroAttr { .. } => MacroKind::Attr, + SyntaxExtensionKind::Derive(..) | + SyntaxExtensionKind::LegacyDerive(..) => MacroKind::Derive, + } + } - /// A declarative macro, e.g., `macro m() {}`. - DeclMacro { - expander: Box<dyn TTMacroExpander + sync::Sync + sync::Send>, - def_info: Option<(ast::NodeId, Span)>, - is_transparent: bool, - edition: Edition, + /// Constructs a syntax extension with default properties. + pub fn default(kind: SyntaxExtensionKind, edition: Edition) -> SyntaxExtension { + SyntaxExtension { + span: DUMMY_SP, + default_transparency: kind.default_transparency(), + allow_internal_unstable: None, + allow_internal_unsafe: false, + local_inner_macros: false, + stability: None, + deprecation: None, + helper_attrs: Vec::new(), + edition, + is_builtin: false, + is_derive_copy: false, + kind, + } } -} -impl SyntaxExtension { - /// Returns which kind of macro calls this syntax extension. - pub fn kind(&self) -> MacroKind { - match *self { - SyntaxExtension::DeclMacro { .. } | - SyntaxExtension::NormalTT { .. } | - SyntaxExtension::IdentTT { .. } | - SyntaxExtension::ProcMacro { .. } => - MacroKind::Bang, - SyntaxExtension::NonMacroAttr { .. } | - SyntaxExtension::MultiDecorator(..) | - SyntaxExtension::MultiModifier(..) | - SyntaxExtension::AttrProcMacro(..) => - MacroKind::Attr, - SyntaxExtension::ProcMacroDerive(..) | - SyntaxExtension::BuiltinDerive(..) => - MacroKind::Derive, + pub fn dummy_bang(edition: Edition) -> SyntaxExtension { + fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: &[TokenTree]) + -> Box<dyn MacResult + 'cx> { + DummyResult::any(span) } + SyntaxExtension::default(SyntaxExtensionKind::LegacyBang(Box::new(expander)), edition) } - pub fn default_transparency(&self) -> Transparency { - match *self { - SyntaxExtension::ProcMacro { .. } | - SyntaxExtension::AttrProcMacro(..) | - SyntaxExtension::ProcMacroDerive(..) | - SyntaxExtension::DeclMacro { is_transparent: false, .. } => Transparency::Opaque, - SyntaxExtension::DeclMacro { is_transparent: true, .. } => Transparency::Transparent, - _ => Transparency::SemiTransparent, + pub fn dummy_derive(edition: Edition) -> SyntaxExtension { + fn expander(_: &mut ExtCtxt<'_>, _: Span, _: &ast::MetaItem, _: Annotatable) + -> Vec<Annotatable> { + Vec::new() } + SyntaxExtension::default(SyntaxExtensionKind::Derive(Box::new(expander)), edition) } - pub fn edition(&self) -> Edition { - match *self { - SyntaxExtension::NormalTT { edition, .. } | - SyntaxExtension::DeclMacro { edition, .. } | - SyntaxExtension::ProcMacro { edition, .. } | - SyntaxExtension::AttrProcMacro(.., edition) | - SyntaxExtension::ProcMacroDerive(.., edition) => edition, - // Unstable legacy stuff - SyntaxExtension::NonMacroAttr { .. } | - SyntaxExtension::IdentTT { .. } | - SyntaxExtension::MultiDecorator(..) | - SyntaxExtension::MultiModifier(..) | - SyntaxExtension::BuiltinDerive(..) => hygiene::default_edition(), + pub fn non_macro_attr(mark_used: bool, edition: Edition) -> SyntaxExtension { + SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr { mark_used }, edition) + } + + pub fn expn_info(&self, call_site: Span, descr: Symbol) -> ExpnInfo { + ExpnInfo { + call_site, + kind: ExpnKind::Macro(self.macro_kind(), descr), + def_site: self.span, + default_transparency: self.default_transparency, + allow_internal_unstable: self.allow_internal_unstable.clone(), + allow_internal_unsafe: self.allow_internal_unsafe, + local_inner_macros: self.local_inner_macros, + edition: self.edition, } } } pub type NamedSyntaxExtension = (Name, SyntaxExtension); +/// Error type that denotes indeterminacy. +pub struct Indeterminate; + +bitflags::bitflags! { + /// Built-in derives that need some extra tracking beyond the usual macro functionality. + #[derive(Default)] + pub struct SpecialDerives: u8 { + const PARTIAL_EQ = 1 << 0; + const EQ = 1 << 1; + const COPY = 1 << 2; + } +} + pub trait Resolver { fn next_node_id(&mut self) -> ast::NodeId; - fn get_module_scope(&mut self, id: ast::NodeId) -> Mark; - fn resolve_dollar_crates(&mut self, fragment: &AstFragment); - fn visit_ast_fragment_with_placeholders(&mut self, mark: Mark, fragment: &AstFragment, - derives: &[Mark]); - fn add_builtin(&mut self, ident: ast::Ident, ext: Lrc<SyntaxExtension>); + fn get_module_scope(&mut self, id: ast::NodeId) -> ExpnId; + + fn resolve_dollar_crates(&mut self); + fn visit_ast_fragment_with_placeholders(&mut self, expn_id: ExpnId, fragment: &AstFragment, + derives: &[ExpnId]); + fn register_builtin_macro(&mut self, ident: ast::Ident, ext: SyntaxExtension); fn resolve_imports(&mut self); - fn resolve_macro_invocation(&mut self, invoc: &Invocation, invoc_id: Mark, force: bool) - -> Result<Option<Lrc<SyntaxExtension>>, Determinacy>; - fn resolve_macro_path(&mut self, path: &ast::Path, kind: MacroKind, invoc_id: Mark, - derives_in_scope: Vec<ast::Path>, force: bool) - -> Result<Lrc<SyntaxExtension>, Determinacy>; + fn resolve_macro_invocation(&mut self, invoc: &Invocation, invoc_id: ExpnId, force: bool) + -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate>; fn check_unused_macros(&self); -} - -#[derive(Copy, Clone, PartialEq, Debug)] -pub enum Determinacy { - Determined, - Undetermined, -} -impl Determinacy { - pub fn determined(determined: bool) -> Determinacy { - if determined { Determinacy::Determined } else { Determinacy::Undetermined } - } -} - -pub struct DummyResolver; - -impl Resolver for DummyResolver { - fn next_node_id(&mut self) -> ast::NodeId { ast::DUMMY_NODE_ID } - fn get_module_scope(&mut self, _id: ast::NodeId) -> Mark { Mark::root() } - - fn resolve_dollar_crates(&mut self, _fragment: &AstFragment) {} - fn visit_ast_fragment_with_placeholders(&mut self, _invoc: Mark, _fragment: &AstFragment, - _derives: &[Mark]) {} - fn add_builtin(&mut self, _ident: ast::Ident, _ext: Lrc<SyntaxExtension>) {} - - fn resolve_imports(&mut self) {} - fn resolve_macro_invocation(&mut self, _invoc: &Invocation, _invoc_id: Mark, _force: bool) - -> Result<Option<Lrc<SyntaxExtension>>, Determinacy> { - Err(Determinacy::Determined) - } - fn resolve_macro_path(&mut self, _path: &ast::Path, _kind: MacroKind, _invoc_id: Mark, - _derives_in_scope: Vec<ast::Path>, _force: bool) - -> Result<Lrc<SyntaxExtension>, Determinacy> { - Err(Determinacy::Determined) - } - fn check_unused_macros(&self) {} + fn has_derives(&self, expn_id: ExpnId, derives: SpecialDerives) -> bool; + fn add_derives(&mut self, expn_id: ExpnId, derives: SpecialDerives); } #[derive(Clone)] @@ -796,11 +725,11 @@ pub struct ModuleData { #[derive(Clone)] pub struct ExpansionData { - pub mark: Mark, + pub id: ExpnId, pub depth: usize, pub module: Rc<ModuleData>, pub directory_ownership: DirectoryOwnership, - pub crate_span: Option<Span>, + pub prior_type_ascription: Option<(Span, bool)>, } /// One of these is made during expansion and incrementally updated as we go; @@ -826,11 +755,11 @@ impl<'a> ExtCtxt<'a> { root_path: PathBuf::new(), resolver, current_expansion: ExpansionData { - mark: Mark::root(), + id: ExpnId::root(), depth: 0, module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }), directory_ownership: DirectoryOwnership::Owned { relative: None }, - crate_span: None, + prior_type_ascription: None, }, expansions: FxHashMap::default(), } @@ -848,19 +777,19 @@ impl<'a> ExtCtxt<'a> { } pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> { - parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect()) + parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect(), MACRO_ARGUMENTS) } pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() } pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess } pub fn cfg(&self) -> &ast::CrateConfig { &self.parse_sess.config } pub fn call_site(&self) -> Span { - match self.current_expansion.mark.expn_info() { + match self.current_expansion.id.expn_info() { Some(expn_info) => expn_info.call_site, None => DUMMY_SP, } } pub fn backtrace(&self) -> SyntaxContext { - SyntaxContext::empty().apply_mark(self.current_expansion.mark) + SyntaxContext::empty().apply_mark(self.current_expansion.id) } /// Returns span for the macro which originally caused the current expansion to happen. @@ -870,8 +799,8 @@ impl<'a> ExtCtxt<'a> { let mut ctxt = self.backtrace(); let mut last_macro = None; loop { - if ctxt.outer().expn_info().map_or(None, |info| { - if info.format.name() == "include" { + if ctxt.outer_expn_info().map_or(None, |info| { + if info.kind.descr() == sym::include { // Stop going up the backtrace once include! is encountered return None; } @@ -967,10 +896,10 @@ impl<'a> ExtCtxt<'a> { pub fn ident_of(&self, st: &str) -> ast::Ident { ast::Ident::from_str(st) } - pub fn std_path(&self, components: &[&str]) -> Vec<ast::Ident> { - let def_site = DUMMY_SP.apply_mark(self.current_expansion.mark); - iter::once(Ident::new(keywords::DollarCrate.name(), def_site)) - .chain(components.iter().map(|s| self.ident_of(s))) + pub fn std_path(&self, components: &[Symbol]) -> Vec<ast::Ident> { + let def_site = DUMMY_SP.apply_mark(self.current_expansion.id); + iter::once(Ident::new(kw::DollarCrate, def_site)) + .chain(components.iter().map(|&s| Ident::with_empty_ctxt(s))) .collect() } pub fn name_of(&self, st: &str) -> ast::Name { @@ -980,6 +909,31 @@ impl<'a> ExtCtxt<'a> { pub fn check_unused_macros(&self) { self.resolver.check_unused_macros(); } + + /// Resolve a path mentioned inside Rust code. + /// + /// This unifies the logic used for resolving `include_X!`, and `#[doc(include)]` file paths. + /// + /// Returns an absolute path to the file that `path` refers to. + pub fn resolve_path(&self, path: impl Into<PathBuf>, span: Span) -> PathBuf { + let path = path.into(); + + // Relative paths are resolved relative to the file in which they are found + // after macro expansion (that is, they are unhygienic). + if !path.is_absolute() { + let callsite = span.source_callsite(); + let mut result = match self.source_map().span_to_unmapped_path(callsite) { + FileName::Real(path) => path, + FileName::DocTest(path, _) => path, + other => panic!("cannot resolve relative path in non-file source `{}`", other), + }; + result.pop(); + result.push(path); + result + } else { + path + } + } } /// Extracts a string literal from the macro expanded version of `expr`, @@ -991,13 +945,14 @@ pub fn expr_to_spanned_string<'a>( err_msg: &str, ) -> Result<Spanned<(Symbol, ast::StrStyle)>, Option<DiagnosticBuilder<'a>>> { // Update `expr.span`'s ctxt now in case expr is an `include!` macro invocation. - expr.span = expr.span.apply_mark(cx.current_expansion.mark); + expr.span = expr.span.apply_mark(cx.current_expansion.id); // we want to be able to handle e.g., `concat!("foo", "bar")` cx.expander().visit_expr(&mut expr); Err(match expr.node { ast::ExprKind::Lit(ref l) => match l.node { ast::LitKind::Str(s, style) => return Ok(respan(expr.span, (s, style))), + ast::LitKind::Err(_) => None, _ => Some(cx.struct_span_err(l.span, err_msg)) }, ast::ExprKind::Err => None, diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index e95f0589449..db562840e8d 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -3,307 +3,31 @@ use crate::attr; use crate::source_map::{dummy_spanned, respan, Spanned}; use crate::ext::base::ExtCtxt; use crate::ptr::P; -use crate::symbol::{Symbol, keywords}; +use crate::symbol::{kw, sym, Symbol}; use crate::ThinVec; use rustc_target::spec::abi::Abi; -use syntax_pos::{Pos, Span, DUMMY_SP}; - -pub trait AstBuilder { - // paths - fn path(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path; - fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path; - fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path; - fn path_all(&self, sp: Span, - global: bool, - idents: Vec<ast::Ident>, - args: Vec<ast::GenericArg>, - bindings: Vec<ast::TypeBinding>) - -> ast::Path; - - fn qpath(&self, self_type: P<ast::Ty>, - trait_path: ast::Path, - ident: ast::Ident) - -> (ast::QSelf, ast::Path); - fn qpath_all(&self, self_type: P<ast::Ty>, - trait_path: ast::Path, - ident: ast::Ident, - args: Vec<ast::GenericArg>, - bindings: Vec<ast::TypeBinding>) - -> (ast::QSelf, ast::Path); - - // types and consts - fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy; - - fn ty(&self, span: Span, ty: ast::TyKind) -> P<ast::Ty>; - fn ty_path(&self, path: ast::Path) -> P<ast::Ty>; - fn ty_ident(&self, span: Span, idents: ast::Ident) -> P<ast::Ty>; - fn anon_const(&self, span: Span, expr: ast::ExprKind) -> ast::AnonConst; - fn const_ident(&self, span: Span, idents: ast::Ident) -> ast::AnonConst; - - fn ty_rptr(&self, span: Span, - ty: P<ast::Ty>, - lifetime: Option<ast::Lifetime>, - mutbl: ast::Mutability) -> P<ast::Ty>; - fn ty_ptr(&self, span: Span, - ty: P<ast::Ty>, - mutbl: ast::Mutability) -> P<ast::Ty>; - - fn ty_option(&self, ty: P<ast::Ty>) -> P<ast::Ty>; - fn ty_infer(&self, sp: Span) -> P<ast::Ty>; - - fn typaram(&self, - span: Span, - id: ast::Ident, - attrs: Vec<ast::Attribute>, - bounds: ast::GenericBounds, - default: Option<P<ast::Ty>>) -> ast::GenericParam; - - fn trait_ref(&self, path: ast::Path) -> ast::TraitRef; - fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef; - fn trait_bound(&self, path: ast::Path) -> ast::GenericBound; - fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime; - fn lifetime_def(&self, - span: Span, - ident: ast::Ident, - attrs: Vec<ast::Attribute>, - bounds: ast::GenericBounds) - -> ast::GenericParam; - - // statements - fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt; - fn stmt_semi(&self, expr: P<ast::Expr>) -> ast::Stmt; - fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, ex: P<ast::Expr>) -> ast::Stmt; - fn stmt_let_typed(&self, - sp: Span, - mutbl: bool, - ident: ast::Ident, - typ: P<ast::Ty>, - ex: P<ast::Expr>) - -> ast::Stmt; - fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt; - fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt; - - // blocks - fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block>; - fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block>; - - // expressions - fn expr(&self, span: Span, node: ast::ExprKind) -> P<ast::Expr>; - fn expr_path(&self, path: ast::Path) -> P<ast::Expr>; - fn expr_qpath(&self, span: Span, qself: ast::QSelf, path: ast::Path) -> P<ast::Expr>; - fn expr_ident(&self, span: Span, id: ast::Ident) -> P<ast::Expr>; - - fn expr_self(&self, span: Span) -> P<ast::Expr>; - fn expr_binary(&self, sp: Span, op: ast::BinOpKind, - lhs: P<ast::Expr>, rhs: P<ast::Expr>) -> P<ast::Expr>; - fn expr_deref(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr>; - fn expr_unary(&self, sp: Span, op: ast::UnOp, e: P<ast::Expr>) -> P<ast::Expr>; - - fn expr_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr>; - fn expr_mut_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr>; - fn expr_field_access(&self, span: Span, expr: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr>; - fn expr_tup_field_access(&self, sp: Span, expr: P<ast::Expr>, - idx: usize) -> P<ast::Expr>; - fn expr_call(&self, span: Span, expr: P<ast::Expr>, args: Vec<P<ast::Expr>>) -> P<ast::Expr>; - fn expr_call_ident(&self, span: Span, id: ast::Ident, args: Vec<P<ast::Expr>>) -> P<ast::Expr>; - fn expr_call_global(&self, sp: Span, fn_path: Vec<ast::Ident>, - args: Vec<P<ast::Expr>> ) -> P<ast::Expr>; - fn expr_method_call(&self, span: Span, - expr: P<ast::Expr>, ident: ast::Ident, - args: Vec<P<ast::Expr>> ) -> P<ast::Expr>; - fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr>; - fn expr_cast(&self, sp: Span, expr: P<ast::Expr>, ty: P<ast::Ty>) -> P<ast::Expr>; - - fn field_imm(&self, span: Span, name: Ident, e: P<ast::Expr>) -> ast::Field; - fn expr_struct(&self, span: Span, path: ast::Path, fields: Vec<ast::Field>) -> P<ast::Expr>; - fn expr_struct_ident(&self, span: Span, id: ast::Ident, - fields: Vec<ast::Field>) -> P<ast::Expr>; - - fn expr_lit(&self, sp: Span, lit: ast::LitKind) -> P<ast::Expr>; - - fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr>; - fn expr_isize(&self, sp: Span, i: isize) -> P<ast::Expr>; - fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr>; - fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr>; - fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr>; - fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr>; - - fn expr_vec(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr>; - fn expr_vec_ng(&self, sp: Span) -> P<ast::Expr>; - fn expr_vec_slice(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr>; - fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr>; - - fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr>; - fn expr_none(&self, sp: Span) -> P<ast::Expr>; - - fn expr_break(&self, sp: Span) -> P<ast::Expr>; - - fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr>; - - fn expr_fail(&self, span: Span, msg: Symbol) -> P<ast::Expr>; - fn expr_unreachable(&self, span: Span) -> P<ast::Expr>; - - fn expr_ok(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Expr>; - fn expr_err(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Expr>; - fn expr_try(&self, span: Span, head: P<ast::Expr>) -> P<ast::Expr>; - - fn pat(&self, span: Span, pat: PatKind) -> P<ast::Pat>; - fn pat_wild(&self, span: Span) -> P<ast::Pat>; - fn pat_lit(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Pat>; - fn pat_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Pat>; - - fn pat_ident_binding_mode(&self, - span: Span, - ident: ast::Ident, - bm: ast::BindingMode) -> P<ast::Pat>; - fn pat_path(&self, span: Span, path: ast::Path) -> P<ast::Pat>; - fn pat_tuple_struct(&self, span: Span, path: ast::Path, - subpats: Vec<P<ast::Pat>>) -> P<ast::Pat>; - fn pat_struct(&self, span: Span, path: ast::Path, - field_pats: Vec<Spanned<ast::FieldPat>>) -> P<ast::Pat>; - fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat>; - - fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat>; - fn pat_none(&self, span: Span) -> P<ast::Pat>; - - fn pat_ok(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat>; - fn pat_err(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat>; - - fn arm(&self, span: Span, pats: Vec<P<ast::Pat>>, expr: P<ast::Expr>) -> ast::Arm; - fn arm_unreachable(&self, span: Span) -> ast::Arm; - - fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm> ) -> P<ast::Expr>; - fn expr_if(&self, span: Span, - cond: P<ast::Expr>, then: P<ast::Expr>, els: Option<P<ast::Expr>>) -> P<ast::Expr>; - fn expr_loop(&self, span: Span, block: P<ast::Block>) -> P<ast::Expr>; - - fn lambda_fn_decl(&self, - span: Span, - fn_decl: P<ast::FnDecl>, - body: P<ast::Expr>, - fn_decl_span: Span) - -> P<ast::Expr>; - - fn lambda(&self, span: Span, ids: Vec<ast::Ident>, body: P<ast::Expr>) -> P<ast::Expr>; - fn lambda0(&self, span: Span, body: P<ast::Expr>) -> P<ast::Expr>; - fn lambda1(&self, span: Span, body: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr>; - - fn lambda_stmts(&self, span: Span, ids: Vec<ast::Ident>, - blk: Vec<ast::Stmt>) -> P<ast::Expr>; - fn lambda_stmts_0(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Expr>; - fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, - ident: ast::Ident) -> P<ast::Expr>; - - // items - fn item(&self, span: Span, - name: Ident, attrs: Vec<ast::Attribute> , node: ast::ItemKind) -> P<ast::Item>; +use syntax_pos::{Pos, Span}; - fn arg(&self, span: Span, name: Ident, ty: P<ast::Ty>) -> ast::Arg; - // FIXME unused self - fn fn_decl(&self, inputs: Vec<ast::Arg> , output: ast::FunctionRetTy) -> P<ast::FnDecl>; +// Left so that Cargo tests don't break, this can be removed once those no longer use it +pub trait AstBuilder {} - fn item_fn_poly(&self, - span: Span, - name: Ident, - inputs: Vec<ast::Arg> , - output: P<ast::Ty>, - generics: Generics, - body: P<ast::Block>) -> P<ast::Item>; - fn item_fn(&self, - span: Span, - name: Ident, - inputs: Vec<ast::Arg> , - output: P<ast::Ty>, - body: P<ast::Block>) -> P<ast::Item>; - - fn variant(&self, span: Span, name: Ident, tys: Vec<P<ast::Ty>> ) -> ast::Variant; - fn item_enum_poly(&self, - span: Span, - name: Ident, - enum_definition: ast::EnumDef, - generics: Generics) -> P<ast::Item>; - fn item_enum(&self, span: Span, name: Ident, enum_def: ast::EnumDef) -> P<ast::Item>; - - fn item_struct_poly(&self, - span: Span, - name: Ident, - struct_def: ast::VariantData, - generics: Generics) -> P<ast::Item>; - fn item_struct(&self, span: Span, name: Ident, struct_def: ast::VariantData) -> P<ast::Item>; - - fn item_mod(&self, span: Span, inner_span: Span, - name: Ident, attrs: Vec<ast::Attribute>, - items: Vec<P<ast::Item>>) -> P<ast::Item>; - - fn item_extern_crate(&self, span: Span, name: Ident) -> P<ast::Item>; - - fn item_static(&self, - span: Span, - name: Ident, - ty: P<ast::Ty>, - mutbl: ast::Mutability, - expr: P<ast::Expr>) - -> P<ast::Item>; - - fn item_const(&self, - span: Span, - name: Ident, - ty: P<ast::Ty>, - expr: P<ast::Expr>) - -> P<ast::Item>; - - fn item_ty_poly(&self, - span: Span, - name: Ident, - ty: P<ast::Ty>, - generics: Generics) -> P<ast::Item>; - fn item_ty(&self, span: Span, name: Ident, ty: P<ast::Ty>) -> P<ast::Item>; - - fn attribute(&self, sp: Span, mi: ast::MetaItem) -> ast::Attribute; - - fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem; - - fn meta_list_item_word(&self, sp: Span, w: ast::Name) -> ast::NestedMetaItem; - - fn meta_list(&self, - sp: Span, - name: ast::Name, - mis: Vec<ast::NestedMetaItem> ) - -> ast::MetaItem; - fn meta_name_value(&self, - sp: Span, - name: ast::Name, - value: ast::LitKind) - -> ast::MetaItem; - - fn item_use(&self, sp: Span, - vis: ast::Visibility, vp: P<ast::UseTree>) -> P<ast::Item>; - fn item_use_simple(&self, sp: Span, vis: ast::Visibility, path: ast::Path) -> P<ast::Item>; - fn item_use_simple_(&self, sp: Span, vis: ast::Visibility, - ident: Option<ast::Ident>, path: ast::Path) -> P<ast::Item>; - fn item_use_list(&self, sp: Span, vis: ast::Visibility, - path: Vec<ast::Ident>, imports: &[ast::Ident]) -> P<ast::Item>; - fn item_use_glob(&self, sp: Span, - vis: ast::Visibility, path: Vec<ast::Ident>) -> P<ast::Item>; -} - -impl<'a> AstBuilder for ExtCtxt<'a> { - fn path(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { +impl<'a> ExtCtxt<'a> { + pub fn path(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { self.path_all(span, false, strs, vec![], vec![]) } - fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { + pub fn path_ident(&self, span: Span, id: ast::Ident) -> ast::Path { self.path(span, vec![id]) } - fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { + pub fn path_global(&self, span: Span, strs: Vec<ast::Ident> ) -> ast::Path { self.path_all(span, true, strs, vec![], vec![]) } - fn path_all(&self, + pub fn path_all(&self, span: Span, global: bool, mut idents: Vec<ast::Ident> , args: Vec<ast::GenericArg>, - bindings: Vec<ast::TypeBinding> ) + constraints: Vec<ast::AssocTyConstraint> ) -> ast::Path { assert!(!idents.is_empty()); let add_root = global && !idents[0].is_path_segment_keyword(); @@ -315,8 +39,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> { segments.extend(idents.into_iter().map(|ident| { ast::PathSegment::from_ident(ident.with_span_pos(span)) })); - let args = if !args.is_empty() || !bindings.is_empty() { - ast::AngleBracketedArgs { args, bindings, span }.into() + let args = if !args.is_empty() || !constraints.is_empty() { + ast::AngleBracketedArgs { args, constraints, span }.into() } else { None }; @@ -331,7 +55,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { /// Constructs a qualified path. /// /// Constructs a path like `<self_type as trait_path>::ident`. - fn qpath(&self, + pub fn qpath(&self, self_type: P<ast::Ty>, trait_path: ast::Path, ident: ast::Ident) @@ -342,16 +66,16 @@ impl<'a> AstBuilder for ExtCtxt<'a> { /// Constructs a qualified path. /// /// Constructs a path like `<self_type as trait_path>::ident<'a, T, A = Bar>`. - fn qpath_all(&self, + pub fn qpath_all(&self, self_type: P<ast::Ty>, trait_path: ast::Path, ident: ast::Ident, args: Vec<ast::GenericArg>, - bindings: Vec<ast::TypeBinding>) + constraints: Vec<ast::AssocTyConstraint>) -> (ast::QSelf, ast::Path) { let mut path = trait_path; - let args = if !args.is_empty() || !bindings.is_empty() { - ast::AngleBracketedArgs { args, bindings, span: ident.span }.into() + let args = if !args.is_empty() || !constraints.is_empty() { + ast::AngleBracketedArgs { args, constraints, span: ident.span }.into() } else { None }; @@ -364,14 +88,14 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }, path) } - fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy { + pub fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy { ast::MutTy { ty, mutbl, } } - fn ty(&self, span: Span, ty: ast::TyKind) -> P<ast::Ty> { + pub fn ty(&self, span: Span, ty: ast::TyKind) -> P<ast::Ty> { P(ast::Ty { id: ast::DUMMY_NODE_ID, span, @@ -379,18 +103,18 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn ty_path(&self, path: ast::Path) -> P<ast::Ty> { + pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> { self.ty(path.span, ast::TyKind::Path(None, path)) } // Might need to take bounds as an argument in the future, if you ever want // to generate a bounded existential trait type. - fn ty_ident(&self, span: Span, ident: ast::Ident) + pub fn ty_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Ty> { self.ty_path(self.path_ident(span, ident)) } - fn anon_const(&self, span: Span, expr: ast::ExprKind) -> ast::AnonConst { + pub fn anon_const(&self, span: Span, expr: ast::ExprKind) -> ast::AnonConst { ast::AnonConst { id: ast::DUMMY_NODE_ID, value: P(ast::Expr { @@ -402,11 +126,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { + pub fn const_ident(&self, span: Span, ident: ast::Ident) -> ast::AnonConst { self.anon_const(span, ast::ExprKind::Path(None, self.path_ident(span, ident))) } - fn ty_rptr(&self, + pub fn ty_rptr(&self, span: Span, ty: P<ast::Ty>, lifetime: Option<ast::Lifetime>, @@ -416,7 +140,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ast::TyKind::Rptr(lifetime, self.ty_mt(ty, mutbl))) } - fn ty_ptr(&self, + pub fn ty_ptr(&self, span: Span, ty: P<ast::Ty>, mutbl: ast::Mutability) @@ -425,20 +149,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ast::TyKind::Ptr(self.ty_mt(ty, mutbl))) } - fn ty_option(&self, ty: P<ast::Ty>) -> P<ast::Ty> { - self.ty_path( - self.path_all(DUMMY_SP, - true, - self.std_path(&["option", "Option"]), - vec![ast::GenericArg::Type(ty)], - Vec::new())) - } - - fn ty_infer(&self, span: Span) -> P<ast::Ty> { + pub fn ty_infer(&self, span: Span) -> P<ast::Ty> { self.ty(span, ast::TyKind::Infer) } - fn typaram(&self, + pub fn typaram(&self, span: Span, ident: ast::Ident, attrs: Vec<ast::Attribute>, @@ -455,14 +170,14 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { + pub fn trait_ref(&self, path: ast::Path) -> ast::TraitRef { ast::TraitRef { path, ref_id: ast::DUMMY_NODE_ID, } } - fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { + pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef { ast::PolyTraitRef { bound_generic_params: Vec::new(), trait_ref: self.trait_ref(path), @@ -470,16 +185,16 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { + pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound { ast::GenericBound::Trait(self.poly_trait_ref(path.span, path), ast::TraitBoundModifier::None) } - fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { + pub fn lifetime(&self, span: Span, ident: ast::Ident) -> ast::Lifetime { ast::Lifetime { id: ast::DUMMY_NODE_ID, ident: ident.with_span_pos(span) } } - fn lifetime_def(&self, + pub fn lifetime_def(&self, span: Span, ident: ast::Ident, attrs: Vec<ast::Attribute>, @@ -495,7 +210,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt { + pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt { ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, @@ -503,7 +218,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn stmt_semi(&self, expr: P<ast::Expr>) -> ast::Stmt { + pub fn stmt_semi(&self, expr: P<ast::Expr>) -> ast::Stmt { ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, @@ -511,7 +226,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, + pub fn stmt_let(&self, sp: Span, mutbl: bool, ident: ast::Ident, ex: P<ast::Expr>) -> ast::Stmt { let pat = if mutbl { let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Mutable); @@ -534,7 +249,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn stmt_let_typed(&self, + pub fn stmt_let_typed(&self, sp: Span, mutbl: bool, ident: ast::Ident, @@ -562,8 +277,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - // Generate `let _: Type;`, usually used for type assertions. - fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt { + // Generates `let _: Type;`, which is usually used for type assertions. + pub fn stmt_let_type_only(&self, span: Span, ty: P<ast::Ty>) -> ast::Stmt { let local = P(ast::Local { pat: self.pat_wild(span), ty: Some(ty), @@ -579,7 +294,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt { + pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt { ast::Stmt { id: ast::DUMMY_NODE_ID, node: ast::StmtKind::Item(item), @@ -587,14 +302,14 @@ impl<'a> AstBuilder for ExtCtxt<'a> { } } - fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> { + pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> { self.block(expr.span, vec![ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, node: ast::StmtKind::Expr(expr), }]) } - fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> { + pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> { P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, @@ -603,7 +318,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn expr(&self, span: Span, node: ast::ExprKind) -> P<ast::Expr> { + pub fn expr(&self, span: Span, node: ast::ExprKind) -> P<ast::Expr> { P(ast::Expr { id: ast::DUMMY_NODE_ID, node, @@ -612,61 +327,65 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn expr_path(&self, path: ast::Path) -> P<ast::Expr> { + pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> { self.expr(path.span, ast::ExprKind::Path(None, path)) } - /// Constructs a QPath expression. - fn expr_qpath(&self, span: Span, qself: ast::QSelf, path: ast::Path) -> P<ast::Expr> { + /// Constructs a `QPath` expression. + pub fn expr_qpath(&self, span: Span, qself: ast::QSelf, path: ast::Path) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Path(Some(qself), path)) } - fn expr_ident(&self, span: Span, id: ast::Ident) -> P<ast::Expr> { + pub fn expr_ident(&self, span: Span, id: ast::Ident) -> P<ast::Expr> { self.expr_path(self.path_ident(span, id)) } - fn expr_self(&self, span: Span) -> P<ast::Expr> { - self.expr_ident(span, keywords::SelfLower.ident()) + pub fn expr_self(&self, span: Span) -> P<ast::Expr> { + self.expr_ident(span, Ident::with_empty_ctxt(kw::SelfLower)) } - fn expr_binary(&self, sp: Span, op: ast::BinOpKind, + pub fn expr_binary(&self, sp: Span, op: ast::BinOpKind, lhs: P<ast::Expr>, rhs: P<ast::Expr>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Binary(Spanned { node: op, span: sp }, lhs, rhs)) } - fn expr_deref(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { + pub fn expr_deref(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { self.expr_unary(sp, UnOp::Deref, e) } - fn expr_unary(&self, sp: Span, op: ast::UnOp, e: P<ast::Expr>) -> P<ast::Expr> { + pub fn expr_unary(&self, sp: Span, op: ast::UnOp, e: P<ast::Expr>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Unary(op, e)) } - fn expr_field_access(&self, sp: Span, expr: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr> { + pub fn expr_field_access( + &self, sp: Span, expr: P<ast::Expr>, ident: ast::Ident, + ) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Field(expr, ident.with_span_pos(sp))) } - fn expr_tup_field_access(&self, sp: Span, expr: P<ast::Expr>, idx: usize) -> P<ast::Expr> { + pub fn expr_tup_field_access(&self, sp: Span, expr: P<ast::Expr>, idx: usize) -> P<ast::Expr> { let ident = Ident::from_str(&idx.to_string()).with_span_pos(sp); self.expr(sp, ast::ExprKind::Field(expr, ident)) } - fn expr_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { + pub fn expr_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Immutable, e)) } - fn expr_mut_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { + pub fn expr_mut_addr_of(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::AddrOf(ast::Mutability::Mutable, e)) } - fn expr_call(&self, span: Span, expr: P<ast::Expr>, args: Vec<P<ast::Expr>>) -> P<ast::Expr> { + pub fn expr_call( + &self, span: Span, expr: P<ast::Expr>, args: Vec<P<ast::Expr>>, + ) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Call(expr, args)) } - fn expr_call_ident(&self, span: Span, id: ast::Ident, + pub fn expr_call_ident(&self, span: Span, id: ast::Ident, args: Vec<P<ast::Expr>>) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args)) } - fn expr_call_global(&self, sp: Span, fn_path: Vec<ast::Ident> , + pub fn expr_call_global(&self, sp: Span, fn_path: Vec<ast::Ident> , args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { let pathexpr = self.expr_path(self.path_global(sp, fn_path)); self.expr_call(sp, pathexpr, args) } - fn expr_method_call(&self, span: Span, + pub fn expr_method_call(&self, span: Span, expr: P<ast::Expr>, ident: ast::Ident, mut args: Vec<P<ast::Expr>> ) -> P<ast::Expr> { @@ -674,10 +393,10 @@ impl<'a> AstBuilder for ExtCtxt<'a> { let segment = ast::PathSegment::from_ident(ident.with_span_pos(span)); self.expr(span, ast::ExprKind::MethodCall(segment, args)) } - fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr> { + pub fn expr_block(&self, b: P<ast::Block>) -> P<ast::Expr> { self.expr(b.span, ast::ExprKind::Block(b, None)) } - fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::Field { + pub fn field_imm(&self, span: Span, ident: Ident, e: P<ast::Expr>) -> ast::Field { ast::Field { ident: ident.with_span_pos(span), expr: e, @@ -686,22 +405,25 @@ impl<'a> AstBuilder for ExtCtxt<'a> { attrs: ThinVec::new(), } } - fn expr_struct(&self, span: Span, path: ast::Path, fields: Vec<ast::Field>) -> P<ast::Expr> { + pub fn expr_struct( + &self, span: Span, path: ast::Path, fields: Vec<ast::Field> + ) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Struct(path, fields, None)) } - fn expr_struct_ident(&self, span: Span, + pub fn expr_struct_ident(&self, span: Span, id: ast::Ident, fields: Vec<ast::Field>) -> P<ast::Expr> { self.expr_struct(span, self.path_ident(span, id), fields) } - fn expr_lit(&self, sp: Span, lit: ast::LitKind) -> P<ast::Expr> { - self.expr(sp, ast::ExprKind::Lit(respan(sp, lit))) + pub fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> { + let lit = ast::Lit::from_lit_kind(lit_kind, span); + self.expr(span, ast::ExprKind::Lit(lit)) } - fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { + pub fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { self.expr_lit(span, ast::LitKind::Int(i as u128, ast::LitIntType::Unsigned(ast::UintTy::Usize))) } - fn expr_isize(&self, sp: Span, i: isize) -> P<ast::Expr> { + pub fn expr_isize(&self, sp: Span, i: isize) -> P<ast::Expr> { if i < 0 { let i = (-i) as u128; let lit_ty = ast::LitIntType::Signed(ast::IntTy::Isize); @@ -712,62 +434,59 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ast::LitIntType::Signed(ast::IntTy::Isize))) } } - fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> { + pub fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> { self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U32))) } - fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr> { + pub fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr> { self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U16))) } - fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr> { + pub fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr> { self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U8))) } - fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr> { + pub fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr> { self.expr_lit(sp, ast::LitKind::Bool(value)) } - fn expr_vec(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + pub fn expr_vec(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Array(exprs)) } - fn expr_vec_ng(&self, sp: Span) -> P<ast::Expr> { - self.expr_call_global(sp, self.std_path(&["vec", "Vec", "new"]), + pub fn expr_vec_ng(&self, sp: Span) -> P<ast::Expr> { + self.expr_call_global(sp, self.std_path(&[sym::vec, sym::Vec, sym::new]), Vec::new()) } - fn expr_vec_slice(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + pub fn expr_vec_slice(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { self.expr_addr_of(sp, self.expr_vec(sp, exprs)) } - fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr> { + pub fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr> { self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) } - fn expr_cast(&self, sp: Span, expr: P<ast::Expr>, ty: P<ast::Ty>) -> P<ast::Expr> { + pub fn expr_cast(&self, sp: Span, expr: P<ast::Expr>, ty: P<ast::Ty>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Cast(expr, ty)) } - - fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { - let some = self.std_path(&["option", "Option", "Some"]); + pub fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); self.expr_call_global(sp, some, vec![expr]) } - fn expr_none(&self, sp: Span) -> P<ast::Expr> { - let none = self.std_path(&["option", "Option", "None"]); + pub fn expr_none(&self, sp: Span) -> P<ast::Expr> { + let none = self.std_path(&[sym::option, sym::Option, sym::None]); let none = self.path_global(sp, none); self.expr_path(none) } - - fn expr_break(&self, sp: Span) -> P<ast::Expr> { + pub fn expr_break(&self, sp: Span) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Break(None, None)) } - - fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { + pub fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> { self.expr(sp, ast::ExprKind::Tup(exprs)) } - fn expr_fail(&self, span: Span, msg: Symbol) -> P<ast::Expr> { + pub fn expr_fail(&self, span: Span, msg: Symbol) -> P<ast::Expr> { let loc = self.source_map().lookup_char_pos(span.lo()); let expr_file = self.expr_str(span, Symbol::intern(&loc.file.name.to_string())); let expr_line = self.expr_u32(span, loc.line as u32); @@ -776,144 +495,145 @@ impl<'a> AstBuilder for ExtCtxt<'a> { let expr_loc_ptr = self.expr_addr_of(span, expr_loc_tuple); self.expr_call_global( span, - self.std_path(&["rt", "begin_panic"]), + self.std_path(&[sym::rt, sym::begin_panic]), vec![ self.expr_str(span, msg), expr_loc_ptr]) } - fn expr_unreachable(&self, span: Span) -> P<ast::Expr> { + pub fn expr_unreachable(&self, span: Span) -> P<ast::Expr> { self.expr_fail(span, Symbol::intern("internal error: entered unreachable code")) } - fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { - let ok = self.std_path(&["result", "Result", "Ok"]); + pub fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); self.expr_call_global(sp, ok, vec![expr]) } - fn expr_err(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { - let err = self.std_path(&["result", "Result", "Err"]); + pub fn expr_err(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> { + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); self.expr_call_global(sp, err, vec![expr]) } - fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> { - let ok = self.std_path(&["result", "Result", "Ok"]); + pub fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> { + let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]); let ok_path = self.path_global(sp, ok); - let err = self.std_path(&["result", "Result", "Err"]); + let err = self.std_path(&[sym::result, sym::Result, sym::Err]); let err_path = self.path_global(sp, err); let binding_variable = self.ident_of("__try_var"); let binding_pat = self.pat_ident(sp, binding_variable); let binding_expr = self.expr_ident(sp, binding_variable); - // Ok(__try_var) pattern + // `Ok(__try_var)` pattern let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]); - // Err(__try_var) (pattern and expression resp.) + // `Err(__try_var)` (pattern and expression respectively) let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]); let err_inner_expr = self.expr_call(sp, self.expr_path(err_path), vec![binding_expr.clone()]); - // return Err(__try_var) + // `return Err(__try_var)` let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr))); - // Ok(__try_var) => __try_var + // `Ok(__try_var) => __try_var` let ok_arm = self.arm(sp, vec![ok_pat], binding_expr); - // Err(__try_var) => return Err(__try_var) + // `Err(__try_var) => return Err(__try_var)` let err_arm = self.arm(sp, vec![err_pat], err_expr); - // match head { Ok() => ..., Err() => ... } + // `match head { Ok() => ..., Err() => ... }` self.expr_match(sp, head, vec![ok_arm, err_arm]) } - fn pat(&self, span: Span, pat: PatKind) -> P<ast::Pat> { - P(ast::Pat { id: ast::DUMMY_NODE_ID, node: pat, span: span }) + pub fn pat(&self, span: Span, pat: PatKind) -> P<ast::Pat> { + P(ast::Pat { id: ast::DUMMY_NODE_ID, node: pat, span }) } - fn pat_wild(&self, span: Span) -> P<ast::Pat> { + pub fn pat_wild(&self, span: Span) -> P<ast::Pat> { self.pat(span, PatKind::Wild) } - fn pat_lit(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Pat> { + pub fn pat_lit(&self, span: Span, expr: P<ast::Expr>) -> P<ast::Pat> { self.pat(span, PatKind::Lit(expr)) } - fn pat_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Pat> { + pub fn pat_ident(&self, span: Span, ident: ast::Ident) -> P<ast::Pat> { let binding_mode = ast::BindingMode::ByValue(ast::Mutability::Immutable); self.pat_ident_binding_mode(span, ident, binding_mode) } - fn pat_ident_binding_mode(&self, + pub fn pat_ident_binding_mode(&self, span: Span, ident: ast::Ident, bm: ast::BindingMode) -> P<ast::Pat> { let pat = PatKind::Ident(bm, ident.with_span_pos(span), None); self.pat(span, pat) } - fn pat_path(&self, span: Span, path: ast::Path) -> P<ast::Pat> { + pub fn pat_path(&self, span: Span, path: ast::Path) -> P<ast::Pat> { self.pat(span, PatKind::Path(None, path)) } - fn pat_tuple_struct(&self, span: Span, path: ast::Path, + pub fn pat_tuple_struct(&self, span: Span, path: ast::Path, subpats: Vec<P<ast::Pat>>) -> P<ast::Pat> { - self.pat(span, PatKind::TupleStruct(path, subpats, None)) + self.pat(span, PatKind::TupleStruct(path, subpats)) } - fn pat_struct(&self, span: Span, path: ast::Path, + pub fn pat_struct(&self, span: Span, path: ast::Path, field_pats: Vec<Spanned<ast::FieldPat>>) -> P<ast::Pat> { self.pat(span, PatKind::Struct(path, field_pats, false)) } - fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat> { - self.pat(span, PatKind::Tuple(pats, None)) + pub fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat> { + self.pat(span, PatKind::Tuple(pats)) } - fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&["option", "Option", "Some"]); + pub fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::option, sym::Option, sym::Some]); let path = self.path_global(span, some); self.pat_tuple_struct(span, path, vec![pat]) } - fn pat_none(&self, span: Span) -> P<ast::Pat> { - let some = self.std_path(&["option", "Option", "None"]); + pub fn pat_none(&self, span: Span) -> P<ast::Pat> { + let some = self.std_path(&[sym::option, sym::Option, sym::None]); let path = self.path_global(span, some); self.pat_path(span, path) } - fn pat_ok(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&["result", "Result", "Ok"]); + pub fn pat_ok(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::result, sym::Result, sym::Ok]); let path = self.path_global(span, some); self.pat_tuple_struct(span, path, vec![pat]) } - fn pat_err(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { - let some = self.std_path(&["result", "Result", "Err"]); + pub fn pat_err(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> { + let some = self.std_path(&[sym::result, sym::Result, sym::Err]); let path = self.path_global(span, some); self.pat_tuple_struct(span, path, vec![pat]) } - fn arm(&self, _span: Span, pats: Vec<P<ast::Pat>>, expr: P<ast::Expr>) -> ast::Arm { + pub fn arm(&self, span: Span, pats: Vec<P<ast::Pat>>, expr: P<ast::Expr>) -> ast::Arm { ast::Arm { attrs: vec![], pats, guard: None, body: expr, + span, } } - fn arm_unreachable(&self, span: Span) -> ast::Arm { + pub fn arm_unreachable(&self, span: Span) -> ast::Arm { self.arm(span, vec![self.pat_wild(span)], self.expr_unreachable(span)) } - fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> { + pub fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> { self.expr(span, ast::ExprKind::Match(arg, arms)) } - fn expr_if(&self, span: Span, cond: P<ast::Expr>, + pub fn expr_if(&self, span: Span, cond: P<ast::Expr>, then: P<ast::Expr>, els: Option<P<ast::Expr>>) -> P<ast::Expr> { let els = els.map(|x| self.expr_block(self.block_expr(x))); self.expr(span, ast::ExprKind::If(cond, self.block_expr(then), els)) } - fn expr_loop(&self, span: Span, block: P<ast::Block>) -> P<ast::Expr> { + pub fn expr_loop(&self, span: Span, block: P<ast::Block>) -> P<ast::Expr> { self.expr(span, ast::ExprKind::Loop(block, None)) } - fn lambda_fn_decl(&self, + pub fn lambda_fn_decl(&self, span: Span, fn_decl: P<ast::FnDecl>, body: P<ast::Expr>, @@ -927,7 +647,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { fn_decl_span)) } - fn lambda(&self, + pub fn lambda(&self, span: Span, ids: Vec<ast::Ident>, body: P<ast::Expr>) @@ -948,40 +668,42 @@ impl<'a> AstBuilder for ExtCtxt<'a> { span)) } - fn lambda0(&self, span: Span, body: P<ast::Expr>) -> P<ast::Expr> { + pub fn lambda0(&self, span: Span, body: P<ast::Expr>) -> P<ast::Expr> { self.lambda(span, Vec::new(), body) } - fn lambda1(&self, span: Span, body: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr> { + pub fn lambda1(&self, span: Span, body: P<ast::Expr>, ident: ast::Ident) -> P<ast::Expr> { self.lambda(span, vec![ident], body) } - fn lambda_stmts(&self, + pub fn lambda_stmts(&self, span: Span, ids: Vec<ast::Ident>, stmts: Vec<ast::Stmt>) -> P<ast::Expr> { self.lambda(span, ids, self.expr_block(self.block(span, stmts))) } - fn lambda_stmts_0(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Expr> { + pub fn lambda_stmts_0(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Expr> { self.lambda0(span, self.expr_block(self.block(span, stmts))) } - fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, + pub fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, ident: ast::Ident) -> P<ast::Expr> { self.lambda1(span, self.expr_block(self.block(span, stmts)), ident) } - fn arg(&self, span: Span, ident: ast::Ident, ty: P<ast::Ty>) -> ast::Arg { + pub fn arg(&self, span: Span, ident: ast::Ident, ty: P<ast::Ty>) -> ast::Arg { let arg_pat = self.pat_ident(span, ident); ast::Arg { - ty, + attrs: ThinVec::default(), + id: ast::DUMMY_NODE_ID, pat: arg_pat, - id: ast::DUMMY_NODE_ID + span, + ty, } } - // FIXME unused self - fn fn_decl(&self, inputs: Vec<ast::Arg>, output: ast::FunctionRetTy) -> P<ast::FnDecl> { + // FIXME: unused `self` + pub fn fn_decl(&self, inputs: Vec<ast::Arg>, output: ast::FunctionRetTy) -> P<ast::FnDecl> { P(ast::FnDecl { inputs, output, @@ -989,7 +711,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn item(&self, span: Span, name: Ident, + pub fn item(&self, span: Span, name: Ident, attrs: Vec<ast::Attribute>, node: ast::ItemKind) -> P<ast::Item> { // FIXME: Would be nice if our generated code didn't violate // Rust coding conventions @@ -1004,7 +726,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn item_fn_poly(&self, + pub fn item_fn_poly(&self, span: Span, name: Ident, inputs: Vec<ast::Arg> , @@ -1025,7 +747,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { body)) } - fn item_fn(&self, + pub fn item_fn(&self, span: Span, name: Ident, inputs: Vec<ast::Arg> , @@ -1041,7 +763,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { body) } - fn variant(&self, span: Span, ident: Ident, tys: Vec<P<ast::Ty>> ) -> ast::Variant { + pub fn variant(&self, span: Span, ident: Ident, tys: Vec<P<ast::Ty>> ) -> ast::Variant { let fields: Vec<_> = tys.into_iter().map(|ty| { ast::StructField { span: ty.span, @@ -1062,25 +784,26 @@ impl<'a> AstBuilder for ExtCtxt<'a> { respan(span, ast::Variant_ { ident, + id: ast::DUMMY_NODE_ID, attrs: Vec::new(), data: vdata, disr_expr: None, }) } - fn item_enum_poly(&self, span: Span, name: Ident, + pub fn item_enum_poly(&self, span: Span, name: Ident, enum_definition: ast::EnumDef, generics: Generics) -> P<ast::Item> { self.item(span, name, Vec::new(), ast::ItemKind::Enum(enum_definition, generics)) } - fn item_enum(&self, span: Span, name: Ident, + pub fn item_enum(&self, span: Span, name: Ident, enum_definition: ast::EnumDef) -> P<ast::Item> { self.item_enum_poly(span, name, enum_definition, Generics::default()) } - fn item_struct(&self, span: Span, name: Ident, + pub fn item_struct(&self, span: Span, name: Ident, struct_def: ast::VariantData) -> P<ast::Item> { self.item_struct_poly( span, @@ -1090,12 +813,12 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ) } - fn item_struct_poly(&self, span: Span, name: Ident, + pub fn item_struct_poly(&self, span: Span, name: Ident, struct_def: ast::VariantData, generics: Generics) -> P<ast::Item> { self.item(span, name, Vec::new(), ast::ItemKind::Struct(struct_def, generics)) } - fn item_mod(&self, span: Span, inner_span: Span, name: Ident, + pub fn item_mod(&self, span: Span, inner_span: Span, name: Ident, attrs: Vec<ast::Attribute>, items: Vec<P<ast::Item>>) -> P<ast::Item> { self.item( @@ -1110,11 +833,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { ) } - fn item_extern_crate(&self, span: Span, name: Ident) -> P<ast::Item> { + pub fn item_extern_crate(&self, span: Span, name: Ident) -> P<ast::Item> { self.item(span, name, Vec::new(), ast::ItemKind::ExternCrate(None)) } - fn item_static(&self, + pub fn item_static(&self, span: Span, name: Ident, ty: P<ast::Ty>, @@ -1124,7 +847,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { self.item(span, name, Vec::new(), ast::ItemKind::Static(ty, mutbl, expr)) } - fn item_const(&self, + pub fn item_const(&self, span: Span, name: Ident, ty: P<ast::Ty>, @@ -1133,43 +856,42 @@ impl<'a> AstBuilder for ExtCtxt<'a> { self.item(span, name, Vec::new(), ast::ItemKind::Const(ty, expr)) } - fn item_ty_poly(&self, span: Span, name: Ident, ty: P<ast::Ty>, + pub fn item_ty_poly(&self, span: Span, name: Ident, ty: P<ast::Ty>, generics: Generics) -> P<ast::Item> { - self.item(span, name, Vec::new(), ast::ItemKind::Ty(ty, generics)) + self.item(span, name, Vec::new(), ast::ItemKind::TyAlias(ty, generics)) } - fn item_ty(&self, span: Span, name: Ident, ty: P<ast::Ty>) -> P<ast::Item> { + pub fn item_ty(&self, span: Span, name: Ident, ty: P<ast::Ty>) -> P<ast::Item> { self.item_ty_poly(span, name, ty, Generics::default()) } - fn attribute(&self, sp: Span, mi: ast::MetaItem) -> ast::Attribute { - attr::mk_spanned_attr_outer(sp, attr::mk_attr_id(), mi) + pub fn attribute(&self, mi: ast::MetaItem) -> ast::Attribute { + attr::mk_attr_outer(mi) } - fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { - attr::mk_word_item(Ident::with_empty_ctxt(w).with_span_pos(sp)) + pub fn meta_word(&self, sp: Span, w: ast::Name) -> ast::MetaItem { + attr::mk_word_item(Ident::new(w, sp)) } - fn meta_list_item_word(&self, sp: Span, w: ast::Name) -> ast::NestedMetaItem { - attr::mk_nested_word_item(Ident::with_empty_ctxt(w).with_span_pos(sp)) + pub fn meta_list_item_word(&self, sp: Span, w: ast::Name) -> ast::NestedMetaItem { + attr::mk_nested_word_item(Ident::new(w, sp)) } - fn meta_list(&self, sp: Span, name: ast::Name, mis: Vec<ast::NestedMetaItem>) + pub fn meta_list(&self, sp: Span, name: ast::Name, mis: Vec<ast::NestedMetaItem>) -> ast::MetaItem { - attr::mk_list_item(sp, Ident::with_empty_ctxt(name).with_span_pos(sp), mis) + attr::mk_list_item(Ident::new(name, sp), mis) } - fn meta_name_value(&self, sp: Span, name: ast::Name, value: ast::LitKind) + pub fn meta_name_value(&self, span: Span, name: ast::Name, lit_kind: ast::LitKind) -> ast::MetaItem { - attr::mk_name_value_item(sp, Ident::with_empty_ctxt(name).with_span_pos(sp), - respan(sp, value)) + attr::mk_name_value_item(Ident::new(name, span), lit_kind, span) } - fn item_use(&self, sp: Span, + pub fn item_use(&self, sp: Span, vis: ast::Visibility, vp: P<ast::UseTree>) -> P<ast::Item> { P(ast::Item { id: ast::DUMMY_NODE_ID, - ident: keywords::Invalid.ident(), + ident: Ident::invalid(), attrs: vec![], node: ast::ItemKind::Use(vp), vis, @@ -1178,11 +900,11 @@ impl<'a> AstBuilder for ExtCtxt<'a> { }) } - fn item_use_simple(&self, sp: Span, vis: ast::Visibility, path: ast::Path) -> P<ast::Item> { + pub fn item_use_simple(&self, sp: Span, vis: ast::Visibility, path: ast::Path) -> P<ast::Item> { self.item_use_simple_(sp, vis, None, path) } - fn item_use_simple_(&self, sp: Span, vis: ast::Visibility, + pub fn item_use_simple_(&self, sp: Span, vis: ast::Visibility, rename: Option<ast::Ident>, path: ast::Path) -> P<ast::Item> { self.item_use(sp, vis, P(ast::UseTree { span: sp, @@ -1191,7 +913,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { })) } - fn item_use_list(&self, sp: Span, vis: ast::Visibility, + pub fn item_use_list(&self, sp: Span, vis: ast::Visibility, path: Vec<ast::Ident>, imports: &[ast::Ident]) -> P<ast::Item> { let imports = imports.iter().map(|id| { (ast::UseTree { @@ -1208,7 +930,7 @@ impl<'a> AstBuilder for ExtCtxt<'a> { })) } - fn item_use_glob(&self, sp: Span, + pub fn item_use_glob(&self, sp: Span, vis: ast::Visibility, path: Vec<ast::Ident>) -> P<ast::Item> { self.item_use(sp, vis, P(ast::UseTree { span: sp, diff --git a/src/libsyntax/ext/derive.rs b/src/libsyntax/ext/derive.rs deleted file mode 100644 index 6df369133d0..00000000000 --- a/src/libsyntax/ext/derive.rs +++ /dev/null @@ -1,81 +0,0 @@ -use crate::attr::HasAttrs; -use crate::ast; -use crate::source_map::{hygiene, ExpnInfo, ExpnFormat}; -use crate::ext::base::ExtCtxt; -use crate::ext::build::AstBuilder; -use crate::parse::parser::PathStyle; -use crate::symbol::Symbol; - -use syntax_pos::Span; - -use rustc_data_structures::fx::FxHashSet; - -pub fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> { - let mut result = Vec::new(); - attrs.retain(|attr| { - if attr.path != "derive" { - return true; - } - if !attr.is_meta_item_list() { - cx.span_err(attr.span, - "attribute must be of the form `#[derive(Trait1, Trait2, ...)]`"); - return false; - } - - match attr.parse_list(cx.parse_sess, - |parser| parser.parse_path_allowing_meta(PathStyle::Mod)) { - Ok(ref traits) if traits.is_empty() => { - cx.span_warn(attr.span, "empty trait list in `derive`"); - false - } - Ok(traits) => { - result.extend(traits); - true - } - Err(mut e) => { - e.emit(); - false - } - } - }); - result -} - -pub fn add_derived_markers<T>(cx: &mut ExtCtxt<'_>, span: Span, traits: &[ast::Path], item: &mut T) - where T: HasAttrs, -{ - let (mut names, mut pretty_name) = (FxHashSet::default(), "derive(".to_owned()); - for (i, path) in traits.iter().enumerate() { - if i > 0 { - pretty_name.push_str(", "); - } - pretty_name.push_str(&path.to_string()); - names.insert(unwrap_or!(path.segments.get(0), continue).ident.name); - } - pretty_name.push(')'); - - cx.current_expansion.mark.set_expn_info(ExpnInfo { - call_site: span, - def_site: None, - format: ExpnFormat::MacroAttribute(Symbol::intern(&pretty_name)), - allow_internal_unstable: Some(vec![ - Symbol::intern("rustc_attrs"), - Symbol::intern("structural_match"), - ].into()), - allow_internal_unsafe: false, - local_inner_macros: false, - edition: hygiene::default_edition(), - }); - - let span = span.with_ctxt(cx.backtrace()); - item.visit_attrs(|attrs| { - if names.contains(&Symbol::intern("Eq")) && names.contains(&Symbol::intern("PartialEq")) { - let meta = cx.meta_word(span, Symbol::intern("structural_match")); - attrs.push(cx.attribute(span, meta)); - } - if names.contains(&Symbol::intern("Copy")) { - let meta = cx.meta_word(span, Symbol::intern("rustc_copy_clone_marker")); - attrs.push(cx.attribute(span, meta)); - } - }); -} diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index b805213bb1a..964c81dd466 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1,20 +1,19 @@ use crate::ast::{self, Block, Ident, LitKind, NodeId, PatKind, Path}; use crate::ast::{MacStmtStyle, StmtKind, ItemKind}; use crate::attr::{self, HasAttrs}; -use crate::source_map::{ExpnInfo, MacroBang, MacroAttribute, dummy_spanned, respan}; +use crate::source_map::{dummy_spanned, respan}; use crate::config::StripUnconfigured; use crate::ext::base::*; -use crate::ext::derive::{add_derived_markers, collect_derives}; -use crate::ext::hygiene::{self, Mark, SyntaxContext}; +use crate::ext::proc_macro::collect_derives; +use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnInfo, ExpnKind}; use crate::ext::placeholders::{placeholder, PlaceholderExpander}; use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; use crate::mut_visit::*; use crate::parse::{DirectoryOwnership, PResult, ParseSess}; -use crate::parse::token::{self, Token}; +use crate::parse::token; use crate::parse::parser::Parser; use crate::ptr::P; -use crate::symbol::Symbol; -use crate::symbol::keywords; +use crate::symbol::{sym, Symbol}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::visit::{self, Visitor}; use crate::util::map_in_place::MapInPlace; @@ -22,7 +21,6 @@ use crate::util::map_in_place::MapInPlace; use errors::{Applicability, FatalError}; use smallvec::{smallvec, SmallVec}; use syntax_pos::{Span, DUMMY_SP, FileName}; -use syntax_pos::hygiene::ExpnFormat; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -99,9 +97,9 @@ macro_rules! ast_fragments { } }); } - $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)*)* + $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)* $($(AstFragment::$Kind(ast) => - ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)*)* + ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)* } } @@ -109,10 +107,10 @@ macro_rules! ast_fragments { match *self { AstFragment::OptExpr(Some(ref expr)) => visitor.visit_expr(expr), AstFragment::OptExpr(None) => {} - $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)*)* + $($(AstFragment::$Kind(ref ast) => visitor.$visit_ast(ast),)?)* $($(AstFragment::$Kind(ref ast) => for ast_elt in &ast[..] { visitor.$visit_ast_elt(ast_elt); - })*)* + })?)* } } } @@ -123,10 +121,10 @@ macro_rules! ast_fragments { } $($(fn $mut_visit_ast(&mut self, ast: &mut $AstTy) { visit_clobber(ast, |ast| self.expand_fragment(AstFragment::$Kind(ast)).$make_ast()); - })*)* + })?)* $($(fn $flat_map_ast_elt(&mut self, ast_elt: <$AstTy as IntoIterator>::Item) -> $AstTy { self.expand_fragment(AstFragment::$Kind(smallvec![ast_elt])).$make_ast() - })*)* + })?)* } impl<'a> MacResult for crate::ext::tt::macro_rules::ParserAnyMacro<'a> { @@ -160,8 +158,8 @@ ast_fragments! { } impl AstFragmentKind { - fn dummy(self, span: Span) -> Option<AstFragment> { - self.make_from(DummyResult::any(span)) + fn dummy(self, span: Span) -> AstFragment { + self.make_from(DummyResult::any(span)).expect("couldn't create a dummy AST fragment") } fn expect_from_annotatables<I: IntoIterator<Item = Annotatable>>(self, items: I) @@ -189,23 +187,6 @@ impl AstFragmentKind { } } -fn macro_bang_format(path: &ast::Path) -> ExpnFormat { - // We don't want to format a path using pretty-printing, - // `format!("{}", path)`, because that tries to insert - // line-breaks and is slow. - let mut path_str = String::with_capacity(64); - for (i, segment) in path.segments.iter().enumerate() { - if i != 0 { - path_str.push_str("::"); - } - if segment.ident.name != keywords::PathRoot.name() { - path_str.push_str(&segment.ident.as_str()) - } - } - - MacroBang(Symbol::intern(&path_str)) -} - pub struct Invocation { pub kind: InvocationKind, fragment_kind: AstFragmentKind, @@ -215,13 +196,13 @@ pub struct Invocation { pub enum InvocationKind { Bang { mac: ast::Mac, - ident: Option<Ident>, span: Span, }, Attr { - attr: Option<ast::Attribute>, - traits: Vec<Path>, + attr: ast::Attribute, item: Annotatable, + // Required for resolving derive helper attributes. + derives: Vec<Path>, // We temporarily report errors for attribute macros placed after derives after_derive: bool, }, @@ -229,27 +210,34 @@ pub enum InvocationKind { path: Path, item: Annotatable, }, + /// "Invocation" that contains all derives from an item, + /// broken into multiple `Derive` invocations when expanded. + /// FIXME: Find a way to remove it. + DeriveContainer { + derives: Vec<Path>, + item: Annotatable, + }, } impl Invocation { pub fn span(&self) -> Span { - match self.kind { - InvocationKind::Bang { span, .. } => span, - InvocationKind::Attr { attr: Some(ref attr), .. } => attr.span, - InvocationKind::Attr { attr: None, .. } => DUMMY_SP, - InvocationKind::Derive { ref path, .. } => path.span, + match &self.kind { + InvocationKind::Bang { span, .. } => *span, + InvocationKind::Attr { attr, .. } => attr.span, + InvocationKind::Derive { path, .. } => path.span, + InvocationKind::DeriveContainer { item, .. } => item.span(), } } } -pub struct MacroExpander<'a, 'b:'a> { +pub struct MacroExpander<'a, 'b> { pub cx: &'a mut ExtCtxt<'b>, monotonic: bool, // cf. `cx.monotonic_expander()` } impl<'a, 'b> MacroExpander<'a, 'b> { pub fn new(cx: &'a mut ExtCtxt<'b>, monotonic: bool) -> Self { - MacroExpander { cx: cx, monotonic: monotonic } + MacroExpander { cx, monotonic } } pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate { @@ -263,7 +251,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> { module.directory.pop(); self.cx.root_path = module.directory.clone(); self.cx.current_expansion.module = Rc::new(module); - self.cx.current_expansion.crate_span = Some(krate.span); let orig_mod_span = krate.module.inner; @@ -271,7 +258,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { attrs: krate.attrs, span: krate.span, node: ast::ItemKind::Mod(krate.module), - ident: keywords::Invalid.ident(), + ident: Ident::invalid(), id: ast::DUMMY_NODE_ID, vis: respan(krate.span.shrink_to_lo(), ast::VisibilityKind::Public), tokens: None, @@ -316,7 +303,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { // Unresolved macros produce dummy outputs as a recovery measure. invocations.reverse(); let mut expanded_fragments = Vec::new(); - let mut derives: FxHashMap<Mark, Vec<_>> = FxHashMap::default(); + let mut derives: FxHashMap<ExpnId, Vec<_>> = FxHashMap::default(); let mut undetermined_invocations = Vec::new(); let (mut progress, mut force) = (false, !self.monotonic); loop { @@ -325,98 +312,80 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } else { self.resolve_imports(); if undetermined_invocations.is_empty() { break } - invocations = mem::replace(&mut undetermined_invocations, Vec::new()); + invocations = mem::take(&mut undetermined_invocations); force = !mem::replace(&mut progress, false); continue }; let scope = - if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark }; + if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id }; let ext = match self.cx.resolver.resolve_macro_invocation(&invoc, scope, force) { - Ok(ext) => Some(ext), - Err(Determinacy::Determined) => None, - Err(Determinacy::Undetermined) => { + Ok(ext) => ext, + Err(Indeterminate) => { undetermined_invocations.push(invoc); continue } }; progress = true; - let ExpansionData { depth, mark, .. } = invoc.expansion_data; + let ExpansionData { depth, id: expn_id, .. } = invoc.expansion_data; self.cx.current_expansion = invoc.expansion_data.clone(); + self.cx.current_expansion.id = scope; - self.cx.current_expansion.mark = scope; // FIXME(jseyfried): Refactor out the following logic let (expanded_fragment, new_invocations) = if let Some(ext) = ext { - if let Some(ext) = ext { - let (invoc_fragment_kind, invoc_span) = (invoc.fragment_kind, invoc.span()); - let fragment = self.expand_invoc(invoc, &*ext).unwrap_or_else(|| { - invoc_fragment_kind.dummy(invoc_span).unwrap() - }); - self.collect_invocations(fragment, &[]) - } else if let InvocationKind::Attr { attr: None, traits, item, .. } = invoc.kind { - if !item.derive_allowed() { - let attr = attr::find_by_name(item.attrs(), "derive") - .expect("`derive` attribute should exist"); - let span = attr.span; - let mut err = self.cx.mut_span_err(span, - "`derive` may only be applied to \ - structs, enums and unions"); - if let ast::AttrStyle::Inner = attr.style { - let trait_list = traits.iter() - .map(|t| t.to_string()).collect::<Vec<_>>(); - let suggestion = format!("#[derive({})]", trait_list.join(", ")); - err.span_suggestion( - span, "try an outer attribute", suggestion, - // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT - Applicability::MaybeIncorrect - ); - } - err.emit(); + let fragment = self.expand_invoc(invoc, &ext.kind); + self.collect_invocations(fragment, &[]) + } else if let InvocationKind::DeriveContainer { derives: traits, item } = invoc.kind { + if !item.derive_allowed() { + let attr = attr::find_by_name(item.attrs(), sym::derive) + .expect("`derive` attribute should exist"); + let span = attr.span; + let mut err = self.cx.mut_span_err(span, + "`derive` may only be applied to \ + structs, enums and unions"); + if let ast::AttrStyle::Inner = attr.style { + let trait_list = traits.iter() + .map(|t| t.to_string()).collect::<Vec<_>>(); + let suggestion = format!("#[derive({})]", trait_list.join(", ")); + err.span_suggestion( + span, "try an outer attribute", suggestion, + // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT + Applicability::MaybeIncorrect + ); } + err.emit(); + } - let mut item = self.fully_configure(item); - item.visit_attrs(|attrs| attrs.retain(|a| a.path != "derive")); - let mut item_with_markers = item.clone(); - add_derived_markers(&mut self.cx, item.span(), &traits, &mut item_with_markers); - let derives = derives.entry(invoc.expansion_data.mark).or_default(); - - derives.reserve(traits.len()); - invocations.reserve(traits.len()); - for path in &traits { - let mark = Mark::fresh(self.cx.current_expansion.mark); - derives.push(mark); - let item = match self.cx.resolver.resolve_macro_path( - path, MacroKind::Derive, Mark::root(), Vec::new(), false) { - Ok(ext) => match *ext { - BuiltinDerive(..) => item_with_markers.clone(), - _ => item.clone(), - }, - _ => item.clone(), - }; - invocations.push(Invocation { - kind: InvocationKind::Derive { path: path.clone(), item: item }, - fragment_kind: invoc.fragment_kind, - expansion_data: ExpansionData { - mark, - ..invoc.expansion_data.clone() - }, - }); - } - let fragment = invoc.fragment_kind - .expect_from_annotatables(::std::iter::once(item_with_markers)); - self.collect_invocations(fragment, derives) - } else { - unreachable!() + let mut item = self.fully_configure(item); + item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive)); + let derives = derives.entry(invoc.expansion_data.id).or_default(); + + derives.reserve(traits.len()); + invocations.reserve(traits.len()); + for path in traits { + let expn_id = ExpnId::fresh(self.cx.current_expansion.id, None); + derives.push(expn_id); + invocations.push(Invocation { + kind: InvocationKind::Derive { path, item: item.clone() }, + fragment_kind: invoc.fragment_kind, + expansion_data: ExpansionData { + id: expn_id, + ..invoc.expansion_data.clone() + }, + }); } + let fragment = invoc.fragment_kind + .expect_from_annotatables(::std::iter::once(item)); + self.collect_invocations(fragment, derives) } else { - self.collect_invocations(invoc.fragment_kind.dummy(invoc.span()).unwrap(), &[]) + unreachable!() }; if expanded_fragments.len() < depth { expanded_fragments.push(Vec::new()); } - expanded_fragments[depth - 1].push((mark, expanded_fragment)); + expanded_fragments[depth - 1].push((expn_id, expanded_fragment)); if !self.cx.ecfg.single_step { invocations.extend(new_invocations.into_iter().rev()); } @@ -429,7 +398,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { while let Some(expanded_fragments) = expanded_fragments.pop() { for (mark, expanded_fragment) in expanded_fragments.into_iter().rev() { let derives = derives.remove(&mark).unwrap_or_else(Vec::new); - placeholder_expander.add(NodeId::placeholder_from_mark(mark), + placeholder_expander.add(NodeId::placeholder_from_expn_id(mark), expanded_fragment, derives); } } @@ -447,10 +416,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> { /// them with "placeholders" - dummy macro invocations with specially crafted `NodeId`s. /// Then call into resolver that builds a skeleton ("reduced graph") of the fragment and /// prepares data for resolving paths of macro invocations. - fn collect_invocations(&mut self, mut fragment: AstFragment, derives: &[Mark]) + fn collect_invocations(&mut self, mut fragment: AstFragment, derives: &[ExpnId]) -> (AstFragment, Vec<Invocation>) { // Resolve `$crate`s in the fragment for pretty-printing. - self.cx.resolver.resolve_dollar_crates(&fragment); + self.cx.resolver.resolve_dollar_crates(); let invocations = { let mut collector = InvocationCollector { @@ -468,7 +437,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { if self.monotonic { self.cx.resolver.visit_ast_fragment_with_placeholders( - self.cx.current_expansion.mark, &fragment, derives); + self.cx.current_expansion.id, &fragment, derives); } (fragment, invocations) @@ -506,28 +475,22 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } } - fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtension) -> Option<AstFragment> { - if invoc.fragment_kind == AstFragmentKind::ForeignItems && - !self.cx.ecfg.macros_in_extern_enabled() { - if let SyntaxExtension::NonMacroAttr { .. } = *ext {} else { - emit_feature_err(&self.cx.parse_sess, "macros_in_extern", - invoc.span(), GateIssue::Language, + fn expand_invoc(&mut self, invoc: Invocation, ext: &SyntaxExtensionKind) -> AstFragment { + let (fragment_kind, span) = (invoc.fragment_kind, invoc.span()); + if fragment_kind == AstFragmentKind::ForeignItems && !self.cx.ecfg.macros_in_extern() { + if let SyntaxExtensionKind::NonMacroAttr { .. } = ext {} else { + emit_feature_err(&self.cx.parse_sess, sym::macros_in_extern, + span, GateIssue::Language, "macro invocations in `extern {}` blocks are experimental"); } } - let result = match invoc.kind { - InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext)?, - InvocationKind::Attr { .. } => self.expand_attr_invoc(invoc, ext)?, - InvocationKind::Derive { .. } => self.expand_derive_invoc(invoc, ext)?, - }; - if self.cx.current_expansion.depth > self.cx.ecfg.recursion_limit { - let info = self.cx.current_expansion.mark.expn_info().unwrap(); + let info = self.cx.current_expansion.id.expn_info().unwrap(); let suggested_limit = self.cx.ecfg.recursion_limit * 2; let mut err = self.cx.struct_span_err(info.call_site, &format!("recursion limit reached while expanding the macro `{}`", - info.format.name())); + info.kind.descr())); err.help(&format!( "consider adding a `#![recursion_limit=\"{}\"]` attribute to your crate", suggested_limit)); @@ -536,81 +499,88 @@ impl<'a, 'b> MacroExpander<'a, 'b> { FatalError.raise(); } - Some(result) - } - - fn expand_attr_invoc(&mut self, - invoc: Invocation, - ext: &SyntaxExtension) - -> Option<AstFragment> { - let (attr, mut item) = match invoc.kind { - InvocationKind::Attr { attr, item, .. } => (attr?, item), - _ => unreachable!(), - }; - - if let NonMacroAttr { mark_used: false } = *ext {} else { - // Macro attrs are always used when expanded, - // non-macro attrs are considered used when the field says so. - attr::mark_used(&attr); - } - invoc.expansion_data.mark.set_expn_info(ExpnInfo { - call_site: attr.span, - def_site: None, - format: MacroAttribute(Symbol::intern(&attr.path.to_string())), - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - edition: ext.edition(), - }); - - match *ext { - NonMacroAttr { .. } => { - attr::mark_known(&attr); - item.visit_attrs(|attrs| attrs.push(attr)); - Some(invoc.fragment_kind.expect_from_annotatables(iter::once(item))) - } - MultiModifier(ref mac) => { - let meta = attr.parse_meta(self.cx.parse_sess) - .map_err(|mut e| { e.emit(); }).ok()?; - let item = mac.expand(self.cx, attr.span, &meta, item); - Some(invoc.fragment_kind.expect_from_annotatables(item)) - } - MultiDecorator(ref mac) => { - let mut items = Vec::new(); - let meta = attr.parse_meta(self.cx.parse_sess) - .expect("derive meta should already have been parsed"); - mac.expand(self.cx, attr.span, &meta, &item, &mut |item| items.push(item)); - items.push(item); - Some(invoc.fragment_kind.expect_from_annotatables(items)) - } - AttrProcMacro(ref mac, ..) => { - self.gate_proc_macro_attr_item(attr.span, &item); - let item_tok = TokenTree::Token(DUMMY_SP, Token::Interpolated(Lrc::new(match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), - Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), - Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), - Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), - Annotatable::Expr(expr) => token::NtExpr(expr), - }))).into(); - let input = self.extract_proc_macro_attr_input(attr.tokens, attr.span); - let tok_result = mac.expand(self.cx, attr.span, input, item_tok); - let res = self.parse_ast_fragment(tok_result, invoc.fragment_kind, - &attr.path, attr.span); - self.gate_proc_macro_expansion(attr.span, &res); - res - } - ProcMacroDerive(..) | BuiltinDerive(..) => { - self.cx.span_err(attr.span, &format!("`{}` is a derive mode", attr.path)); - self.cx.trace_macros_diag(); - invoc.fragment_kind.dummy(attr.span) + match invoc.kind { + InvocationKind::Bang { mac, .. } => match ext { + SyntaxExtensionKind::Bang(expander) => { + self.gate_proc_macro_expansion_kind(span, fragment_kind); + let tok_result = expander.expand(self.cx, span, mac.node.stream()); + let result = + self.parse_ast_fragment(tok_result, fragment_kind, &mac.node.path, span); + self.gate_proc_macro_expansion(span, &result); + result + } + SyntaxExtensionKind::LegacyBang(expander) => { + let prev = self.cx.current_expansion.prior_type_ascription; + self.cx.current_expansion.prior_type_ascription = + mac.node.prior_type_ascription; + let tok_result = expander.expand(self.cx, span, mac.node.stream()); + let result = if let Some(result) = fragment_kind.make_from(tok_result) { + result + } else { + let msg = format!("non-{kind} macro in {kind} position: {path}", + kind = fragment_kind.name(), path = mac.node.path); + self.cx.span_err(span, &msg); + self.cx.trace_macros_diag(); + fragment_kind.dummy(span) + }; + self.cx.current_expansion.prior_type_ascription = prev; + result + } + _ => unreachable!() + } + InvocationKind::Attr { attr, mut item, .. } => match ext { + SyntaxExtensionKind::Attr(expander) => { + self.gate_proc_macro_attr_item(span, &item); + let item_tok = TokenTree::token(token::Interpolated(Lrc::new(match item { + Annotatable::Item(item) => token::NtItem(item), + Annotatable::TraitItem(item) => token::NtTraitItem(item.into_inner()), + Annotatable::ImplItem(item) => token::NtImplItem(item.into_inner()), + Annotatable::ForeignItem(item) => token::NtForeignItem(item.into_inner()), + Annotatable::Stmt(stmt) => token::NtStmt(stmt.into_inner()), + Annotatable::Expr(expr) => token::NtExpr(expr), + })), DUMMY_SP).into(); + let input = self.extract_proc_macro_attr_input(attr.tokens, span); + let tok_result = expander.expand(self.cx, span, input, item_tok); + let res = self.parse_ast_fragment(tok_result, fragment_kind, &attr.path, span); + self.gate_proc_macro_expansion(span, &res); + res + } + SyntaxExtensionKind::LegacyAttr(expander) => { + match attr.parse_meta(self.cx.parse_sess) { + Ok(meta) => { + let item = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(item) + } + Err(mut err) => { + err.emit(); + fragment_kind.dummy(span) + } + } + } + SyntaxExtensionKind::NonMacroAttr { mark_used } => { + attr::mark_known(&attr); + if *mark_used { + attr::mark_used(&attr); + } + item.visit_attrs(|attrs| attrs.push(attr)); + fragment_kind.expect_from_annotatables(iter::once(item)) + } + _ => unreachable!() } - _ => { - let msg = &format!("macro `{}` may not be used in attributes", attr.path); - self.cx.span_err(attr.span, msg); - self.cx.trace_macros_diag(); - invoc.fragment_kind.dummy(attr.span) + InvocationKind::Derive { path, item } => match ext { + SyntaxExtensionKind::Derive(expander) | + SyntaxExtensionKind::LegacyDerive(expander) => { + if !item.derive_allowed() { + return fragment_kind.dummy(span); + } + let meta = ast::MetaItem { node: ast::MetaItemKind::Word, span, path }; + let span = span.with_ctxt(self.cx.backtrace()); + let items = expander.expand(self.cx, span, &meta, item); + fragment_kind.expect_from_annotatables(items) + } + _ => unreachable!() } + InvocationKind::DeriveContainer { .. } => unreachable!() } } @@ -626,7 +596,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { None => return TokenStream::empty(), } self.cx.span_err(span, "custom attribute invocations must be \ - of the form #[foo] or #[foo(..)], the macro name must only be \ + of the form `#[foo]` or `#[foo(..)]`, the macro name must only be \ followed by a delimiter token"); TokenStream::empty() } @@ -636,7 +606,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { Annotatable::Item(ref item) => { match item.node { ItemKind::Mod(_) if self.cx.ecfg.proc_macro_hygiene() => return, - ItemKind::Mod(_) => ("modules", "proc_macro_hygiene"), + ItemKind::Mod(_) => ("modules", sym::proc_macro_hygiene), _ => return, } } @@ -645,8 +615,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> { Annotatable::ForeignItem(_) => return, Annotatable::Stmt(_) | Annotatable::Expr(_) if self.cx.ecfg.proc_macro_hygiene() => return, - Annotatable::Stmt(_) => ("statements", "proc_macro_hygiene"), - Annotatable::Expr(_) => ("expressions", "proc_macro_hygiene"), + Annotatable::Stmt(_) => ("statements", sym::proc_macro_hygiene), + Annotatable::Expr(_) => ("expressions", sym::proc_macro_hygiene), }; emit_feature_err( self.cx.parse_sess, @@ -657,14 +627,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> { ); } - fn gate_proc_macro_expansion(&self, span: Span, fragment: &Option<AstFragment>) { + fn gate_proc_macro_expansion(&self, span: Span, fragment: &AstFragment) { if self.cx.ecfg.proc_macro_hygiene() { return } - let fragment = match fragment { - Some(fragment) => fragment, - None => return, - }; fragment.visit_with(&mut DisallowMacros { span, @@ -681,7 +647,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> { if let ast::ItemKind::MacroDef(_) = i.node { emit_feature_err( self.parse_sess, - "proc_macro_hygiene", + sym::proc_macro_hygiene, self.span, GateIssue::Language, "procedural macros cannot expand to macro definitions", @@ -696,178 +662,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } } - /// Expand a macro invocation. Returns the resulting expanded AST fragment. - fn expand_bang_invoc(&mut self, - invoc: Invocation, - ext: &SyntaxExtension) - -> Option<AstFragment> { - let (mark, kind) = (invoc.expansion_data.mark, invoc.fragment_kind); - let (mac, ident, span) = match invoc.kind { - InvocationKind::Bang { mac, ident, span } => (mac, ident, span), - _ => unreachable!(), - }; - let path = &mac.node.path; - - let ident = ident.unwrap_or_else(|| keywords::Invalid.ident()); - let validate_and_set_expn_info = |this: &mut Self, // arg instead of capture - def_site_span: Option<Span>, - allow_internal_unstable, - allow_internal_unsafe, - local_inner_macros, - // can't infer this type - unstable_feature: Option<(Symbol, u32)>, - edition| { - - // feature-gate the macro invocation - if let Some((feature, issue)) = unstable_feature { - let crate_span = this.cx.current_expansion.crate_span.unwrap(); - // don't stability-check macros in the same crate - // (the only time this is null is for syntax extensions registered as macros) - if def_site_span.map_or(false, |def_span| !crate_span.contains(def_span)) - && !span.allows_unstable(&feature.as_str()) - && this.cx.ecfg.features.map_or(true, |feats| { - // macro features will count as lib features - !feats.declared_lib_features.iter().any(|&(feat, _)| feat == feature) - }) { - let explain = format!("macro {}! is unstable", path); - emit_feature_err(this.cx.parse_sess, &*feature.as_str(), span, - GateIssue::Library(Some(issue)), &explain); - this.cx.trace_macros_diag(); - } - } - - if ident.name != keywords::Invalid.name() { - let msg = format!("macro {}! expects no ident argument, given '{}'", path, ident); - this.cx.span_err(path.span, &msg); - this.cx.trace_macros_diag(); - return Err(kind.dummy(span)); - } - mark.set_expn_info(ExpnInfo { - call_site: span, - def_site: def_site_span, - format: macro_bang_format(path), - allow_internal_unstable, - allow_internal_unsafe, - local_inner_macros, - edition, - }); - Ok(()) - }; - - let opt_expanded = match *ext { - DeclMacro { ref expander, def_info, edition, .. } => { - if let Err(dummy_span) = validate_and_set_expn_info(self, def_info.map(|(_, s)| s), - None, false, false, None, - edition) { - dummy_span - } else { - kind.make_from(expander.expand(self.cx, span, mac.node.stream(), None)) - } - } - - NormalTT { - ref expander, - def_info, - ref allow_internal_unstable, - allow_internal_unsafe, - local_inner_macros, - unstable_feature, - edition, - } => { - if let Err(dummy_span) = validate_and_set_expn_info(self, def_info.map(|(_, s)| s), - allow_internal_unstable.clone(), - allow_internal_unsafe, - local_inner_macros, - unstable_feature, - edition) { - dummy_span - } else { - kind.make_from(expander.expand( - self.cx, - span, - mac.node.stream(), - def_info.map(|(_, s)| s), - )) - } - } - - IdentTT { ref expander, span: tt_span, ref allow_internal_unstable } => { - if ident.name == keywords::Invalid.name() { - self.cx.span_err(path.span, - &format!("macro {}! expects an ident argument", path)); - self.cx.trace_macros_diag(); - kind.dummy(span) - } else { - invoc.expansion_data.mark.set_expn_info(ExpnInfo { - call_site: span, - def_site: tt_span, - format: macro_bang_format(path), - allow_internal_unstable: allow_internal_unstable.clone(), - allow_internal_unsafe: false, - local_inner_macros: false, - edition: hygiene::default_edition(), - }); - - let input: Vec<_> = mac.node.stream().into_trees().collect(); - kind.make_from(expander.expand(self.cx, span, ident, input)) - } - } - - MultiDecorator(..) | MultiModifier(..) | - AttrProcMacro(..) | SyntaxExtension::NonMacroAttr { .. } => { - self.cx.span_err(path.span, - &format!("`{}` can only be used in attributes", path)); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - - ProcMacroDerive(..) | BuiltinDerive(..) => { - self.cx.span_err(path.span, &format!("`{}` is a derive mode", path)); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - - SyntaxExtension::ProcMacro { ref expander, ref allow_internal_unstable, edition } => { - if ident.name != keywords::Invalid.name() { - let msg = - format!("macro {}! expects no ident argument, given '{}'", path, ident); - self.cx.span_err(path.span, &msg); - self.cx.trace_macros_diag(); - kind.dummy(span) - } else { - self.gate_proc_macro_expansion_kind(span, kind); - invoc.expansion_data.mark.set_expn_info(ExpnInfo { - call_site: span, - // FIXME procedural macros do not have proper span info - // yet, when they do, we should use it here. - def_site: None, - format: macro_bang_format(path), - // FIXME probably want to follow macro_rules macros here. - allow_internal_unstable: allow_internal_unstable.clone(), - allow_internal_unsafe: false, - local_inner_macros: false, - edition, - }); - - let tok_result = expander.expand(self.cx, span, mac.node.stream()); - let result = self.parse_ast_fragment(tok_result, kind, path, span); - self.gate_proc_macro_expansion(span, &result); - result - } - } - }; - - if opt_expanded.is_some() { - opt_expanded - } else { - let msg = format!("non-{kind} macro in {kind} position: {name}", - name = path.segments[0].ident.name, kind = kind.name()); - self.cx.span_err(path.span, &msg); - self.cx.trace_macros_diag(); - kind.dummy(span) - } - } - fn gate_proc_macro_expansion_kind(&self, span: Span, kind: AstFragmentKind) { let kind = match kind { AstFragmentKind::Expr => "expressions", @@ -885,90 +679,24 @@ impl<'a, 'b> MacroExpander<'a, 'b> { } emit_feature_err( self.cx.parse_sess, - "proc_macro_hygiene", + sym::proc_macro_hygiene, span, GateIssue::Language, &format!("procedural macros cannot be expanded to {}", kind), ); } - /// Expand a derive invocation. Returns the resulting expanded AST fragment. - fn expand_derive_invoc(&mut self, - invoc: Invocation, - ext: &SyntaxExtension) - -> Option<AstFragment> { - let (path, item) = match invoc.kind { - InvocationKind::Derive { path, item } => (path, item), - _ => unreachable!(), - }; - if !item.derive_allowed() { - return None; - } - - let pretty_name = Symbol::intern(&format!("derive({})", path)); - let span = path.span; - let attr = ast::Attribute { - path, span, - tokens: TokenStream::empty(), - // irrelevant: - id: ast::AttrId(0), style: ast::AttrStyle::Outer, is_sugared_doc: false, - }; - - let mut expn_info = ExpnInfo { - call_site: span, - def_site: None, - format: MacroAttribute(pretty_name), - allow_internal_unstable: None, - allow_internal_unsafe: false, - local_inner_macros: false, - edition: ext.edition(), - }; - - match *ext { - ProcMacroDerive(ref ext, ..) => { - invoc.expansion_data.mark.set_expn_info(expn_info); - let span = span.with_ctxt(self.cx.backtrace()); - let dummy = ast::MetaItem { // FIXME(jseyfried) avoid this - ident: Path::from_ident(keywords::Invalid.ident()), - span: DUMMY_SP, - node: ast::MetaItemKind::Word, - }; - let items = ext.expand(self.cx, span, &dummy, item); - Some(invoc.fragment_kind.expect_from_annotatables(items)) - } - BuiltinDerive(func) => { - expn_info.allow_internal_unstable = Some(vec![ - Symbol::intern("rustc_attrs"), - Symbol::intern("derive_clone_copy"), - Symbol::intern("derive_eq"), - Symbol::intern("libstd_sys_internals"), // RustcDeserialize and RustcSerialize - ].into()); - invoc.expansion_data.mark.set_expn_info(expn_info); - let span = span.with_ctxt(self.cx.backtrace()); - let mut items = Vec::new(); - func(self.cx, span, &attr.meta()?, &item, &mut |a| items.push(a)); - Some(invoc.fragment_kind.expect_from_annotatables(items)) - } - _ => { - let msg = &format!("macro `{}` may not be used for derive attributes", attr.path); - self.cx.span_err(span, msg); - self.cx.trace_macros_diag(); - invoc.fragment_kind.dummy(span) - } - } - } - fn parse_ast_fragment(&mut self, toks: TokenStream, kind: AstFragmentKind, path: &Path, span: Span) - -> Option<AstFragment> { + -> AstFragment { let mut parser = self.cx.new_parser_from_tts(&toks.into_trees().collect::<Vec<_>>()); match parser.parse_ast_fragment(kind, false) { Ok(fragment) => { parser.ensure_complete_parse(path, kind.name(), span); - Some(fragment) + fragment } Err(mut err) => { err.set_span(span); @@ -1008,7 +736,7 @@ impl<'a> Parser<'a> { AstFragmentKind::ForeignItems => { let mut items = SmallVec::new(); while self.token != token::Eof { - items.push(self.parse_foreign_item()?); + items.push(self.parse_foreign_item(DUMMY_SP)?); } AstFragment::ForeignItems(items) } @@ -1041,7 +769,7 @@ impl<'a> Parser<'a> { let msg = format!("macro expansion ignores token `{}` and any following", self.this_token_to_string()); // Avoid emitting backtrace info twice. - let def_site_span = self.span.with_ctxt(SyntaxContext::empty()); + let def_site_span = self.token.span.with_ctxt(SyntaxContext::empty()); let mut err = self.diagnostic().struct_span_err(def_site_span, &msg); err.span_label(span, "caused by the macro expansion here"); let msg = format!( @@ -1069,7 +797,7 @@ impl<'a> Parser<'a> { } } -struct InvocationCollector<'a, 'b: 'a> { +struct InvocationCollector<'a, 'b> { cx: &'a mut ExtCtxt<'b>, cfg: StripUnconfigured<'a>, invocations: Vec<Invocation>, @@ -1078,47 +806,60 @@ struct InvocationCollector<'a, 'b: 'a> { impl<'a, 'b> InvocationCollector<'a, 'b> { fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment { - let mark = Mark::fresh(self.cx.current_expansion.mark); + // Expansion info for all the collected invocations is set upon their resolution, + // with exception of the derive container case which is not resolved and can get + // its expansion info immediately. + let expn_info = match &kind { + InvocationKind::DeriveContainer { item, .. } => Some(ExpnInfo::default( + ExpnKind::Macro(MacroKind::Attr, sym::derive), + item.span(), self.cx.parse_sess.edition, + )), + _ => None, + }; + let expn_id = ExpnId::fresh(self.cx.current_expansion.id, expn_info); self.invocations.push(Invocation { kind, fragment_kind, expansion_data: ExpansionData { - mark, + id: expn_id, depth: self.cx.current_expansion.depth + 1, ..self.cx.current_expansion.clone() }, }); - placeholder(fragment_kind, NodeId::placeholder_from_mark(mark)) + placeholder(fragment_kind, NodeId::placeholder_from_expn_id(expn_id)) } fn collect_bang(&mut self, mac: ast::Mac, span: Span, kind: AstFragmentKind) -> AstFragment { - self.collect(kind, InvocationKind::Bang { mac: mac, ident: None, span: span }) + self.collect(kind, InvocationKind::Bang { mac, span }) } fn collect_attr(&mut self, attr: Option<ast::Attribute>, - traits: Vec<Path>, + derives: Vec<Path>, item: Annotatable, kind: AstFragmentKind, after_derive: bool) -> AstFragment { - self.collect(kind, InvocationKind::Attr { attr, traits, item, after_derive }) + self.collect(kind, match attr { + Some(attr) => InvocationKind::Attr { attr, item, derives, after_derive }, + None => InvocationKind::DeriveContainer { derives, item }, + }) } fn find_attr_invoc(&self, attrs: &mut Vec<ast::Attribute>, after_derive: &mut bool) -> Option<ast::Attribute> { let attr = attrs.iter() .position(|a| { - if a.path == "derive" { + if a.path == sym::derive { *after_derive = true; } !attr::is_known(a) && !is_builtin_attr(a) }) .map(|i| attrs.remove(i)); if let Some(attr) = &attr { - if !self.cx.ecfg.enable_custom_inner_attributes() && - attr.style == ast::AttrStyle::Inner && attr.path != "test" { - emit_feature_err(&self.cx.parse_sess, "custom_inner_attributes", + if !self.cx.ecfg.custom_inner_attributes() && + attr.style == ast::AttrStyle::Inner && attr.path != sym::test { + emit_feature_err(&self.cx.parse_sess, sym::custom_inner_attributes, attr.span, GateIssue::Language, "non-builtin inner attributes are unstable"); } @@ -1167,7 +908,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> { self.check_attribute_inner(attr, features); // macros are expanded before any lint passes so this warning has to be hardcoded - if attr.path == "derive" { + if attr.path == sym::derive { self.cx.struct_span_warn(attr.span, "`#[derive]` does nothing on macro invocations") .note("this may become a hard error in a future release") .emit(); @@ -1327,18 +1068,14 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { ast::ItemKind::Mac(..) => { self.check_attributes(&item.attrs); item.and_then(|item| match item.node { - ItemKind::Mac(mac) => { - self.collect(AstFragmentKind::Items, InvocationKind::Bang { - mac, - ident: Some(item.ident), - span: item.span, - }).make_items() - } + ItemKind::Mac(mac) => self.collect( + AstFragmentKind::Items, InvocationKind::Bang { mac, span: item.span } + ).make_items(), _ => unreachable!(), }) } ast::ItemKind::Mod(ast::Mod { inner, .. }) => { - if item.ident == keywords::Invalid.ident() { + if item.ident == Ident::invalid() { return noop_flat_map_item(item, self); } @@ -1352,7 +1089,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { let inline_module = item.span.contains(inner) || inner.is_dummy(); if inline_module { - if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, "path") { + if let Some(path) = attr::first_attr_value_str_by_name(&item.attrs, sym::path) { self.cx.current_expansion.directory_ownership = DirectoryOwnership::Owned { relative: None }; module.directory.push(&*path.as_str()); @@ -1477,27 +1214,27 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { } } - fn visit_generic_param(&mut self, param: &mut ast::GenericParam) { - self.cfg.disallow_cfg_on_generic_param(¶m); - noop_visit_generic_param(param, self) + fn visit_generic_params(&mut self, params: &mut Vec<ast::GenericParam>) { + self.cfg.configure_generic_params(params); + noop_visit_generic_params(params, self); } fn visit_attribute(&mut self, at: &mut ast::Attribute) { // turn `#[doc(include="filename")]` attributes into `#[doc(include(file="filename", // contents="file contents")]` attributes - if !at.check_name("doc") { + if !at.check_name(sym::doc) { return noop_visit_attribute(at, self); } if let Some(list) = at.meta_item_list() { - if !list.iter().any(|it| it.check_name("include")) { + if !list.iter().any(|it| it.check_name(sym::include)) { return noop_visit_attribute(at, self); } let mut items = vec![]; for mut it in list { - if !it.check_name("include") { + if !it.check_name(sym::include) { items.push({ noop_visit_meta_list_item(&mut it, self); it }); continue; } @@ -1510,7 +1247,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { return noop_visit_attribute(at, self); } - let filename = self.cx.root_path.join(file.to_string()); + let filename = self.cx.resolve_path(&*file.as_str(), it.span()); match fs::read_to_string(&filename) { Ok(src) => { let src_interned = Symbol::intern(&src); @@ -1520,23 +1257,23 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { self.cx.source_map().new_source_file(filename.into(), src); let include_info = vec![ - dummy_spanned(ast::NestedMetaItemKind::MetaItem( + ast::NestedMetaItem::MetaItem( attr::mk_name_value_item_str( - Ident::from_str("file"), + Ident::with_empty_ctxt(sym::file), dummy_spanned(file), ), - )), - dummy_spanned(ast::NestedMetaItemKind::MetaItem( + ), + ast::NestedMetaItem::MetaItem( attr::mk_name_value_item_str( - Ident::from_str("contents"), + Ident::with_empty_ctxt(sym::contents), dummy_spanned(src_interned), ), - )), + ), ]; - let include_ident = Ident::from_str("include"); - let item = attr::mk_list_item(DUMMY_SP, include_ident, include_info); - items.push(dummy_spanned(ast::NestedMetaItemKind::MetaItem(item))); + let include_ident = Ident::with_empty_ctxt(sym::include); + let item = attr::mk_list_item(include_ident, include_info); + items.push(ast::NestedMetaItem::MetaItem(item)); } Err(e) => { let lit = it @@ -1559,17 +1296,13 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { ); err.span_label(lit.span, "couldn't read file"); - if e.kind() == ErrorKind::NotFound { - err.help("external doc paths are relative to the crate root"); - } - err.emit(); } } } } else { let mut err = self.cx.struct_span_err( - it.span, + it.span(), &format!("expected path to external documentation"), ); @@ -1590,7 +1323,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { }; err.span_suggestion( - it.span, + it.span(), "provide a file path with `=`", format!("include = \"{}\"", path), applicability, @@ -1600,11 +1333,15 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { } } - let meta = attr::mk_list_item(DUMMY_SP, Ident::from_str("doc"), items); - match at.style { - ast::AttrStyle::Inner => *at = attr::mk_spanned_attr_inner(at.span, at.id, meta), - ast::AttrStyle::Outer => *at = attr::mk_spanned_attr_outer(at.span, at.id, meta), - } + let meta = attr::mk_list_item(Ident::with_empty_ctxt(sym::doc), items); + *at = attr::Attribute { + span: at.span, + id: at.id, + style: at.style, + path: meta.path, + tokens: meta.node.tokens(meta.span), + is_sugared_doc: false, + }; } else { noop_visit_attribute(at, self) } @@ -1616,6 +1353,11 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { *id = self.cx.resolver.next_node_id() } } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P<ast::FnDecl>) { + self.cfg.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } } pub struct ExpansionConfig<'feat> { @@ -1628,19 +1370,6 @@ pub struct ExpansionConfig<'feat> { pub keep_macs: bool, } -macro_rules! feature_tests { - ($( fn $getter:ident = $field:ident, )*) => { - $( - pub fn $getter(&self) -> bool { - match self.features { - Some(&Features { $field: true, .. }) => true, - _ => false, - } - } - )* - } -} - impl<'feat> ExpansionConfig<'feat> { pub fn default(crate_name: String) -> ExpansionConfig<'static> { ExpansionConfig { @@ -1654,29 +1383,20 @@ impl<'feat> ExpansionConfig<'feat> { } } - feature_tests! { - fn enable_asm = asm, - fn enable_custom_test_frameworks = custom_test_frameworks, - fn enable_global_asm = global_asm, - fn enable_log_syntax = log_syntax, - fn enable_concat_idents = concat_idents, - fn enable_trace_macros = trace_macros, - fn enable_allow_internal_unstable = allow_internal_unstable, - fn enable_format_args_nl = format_args_nl, - fn macros_in_extern_enabled = macros_in_extern, - fn proc_macro_hygiene = proc_macro_hygiene, + fn macros_in_extern(&self) -> bool { + self.features.map_or(false, |features| features.macros_in_extern) } - - fn enable_custom_inner_attributes(&self) -> bool { - self.features.map_or(false, |features| { - features.custom_inner_attributes || features.custom_attribute || features.rustc_attrs - }) + fn proc_macro_hygiene(&self) -> bool { + self.features.map_or(false, |features| features.proc_macro_hygiene) + } + fn custom_inner_attributes(&self) -> bool { + self.features.map_or(false, |features| features.custom_inner_attributes) } } // A Marker adds the given mark to the syntax context. #[derive(Debug)] -pub struct Marker(pub Mark); +pub struct Marker(pub ExpnId); impl MutVisitor for Marker { fn visit_span(&mut self, span: &mut Span) { diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs index 3e60dd81a3b..b2b17b0fb28 100644 --- a/src/libsyntax/ext/placeholders.rs +++ b/src/libsyntax/ext/placeholders.rs @@ -2,11 +2,10 @@ use crate::ast::{self, NodeId}; use crate::source_map::{DUMMY_SP, dummy_spanned}; use crate::ext::base::ExtCtxt; use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::ext::hygiene::Mark; +use crate::ext::hygiene::ExpnId; use crate::tokenstream::TokenStream; use crate::mut_visit::*; use crate::ptr::P; -use crate::symbol::keywords; use crate::ThinVec; use smallvec::{smallvec, SmallVec}; @@ -19,10 +18,11 @@ pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, tts: TokenStream::empty().into(), delim: ast::MacDelimiter::Brace, + prior_type_ascription: None, }) } - let ident = keywords::Invalid.ident(); + let ident = ast::Ident::invalid(); let attrs = Vec::new(); let generics = ast::Generics::default(); let vis = dummy_spanned(ast::VisibilityKind::Inherited); @@ -70,7 +70,7 @@ pub fn placeholder(kind: AstFragmentKind, id: ast::NodeId) -> AstFragment { } } -pub struct PlaceholderExpander<'a, 'b: 'a> { +pub struct PlaceholderExpander<'a, 'b> { expanded_fragments: FxHashMap<ast::NodeId, AstFragment>, cx: &'a mut ExtCtxt<'b>, monotonic: bool, @@ -85,11 +85,11 @@ impl<'a, 'b> PlaceholderExpander<'a, 'b> { } } - pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, derives: Vec<Mark>) { + pub fn add(&mut self, id: ast::NodeId, mut fragment: AstFragment, derives: Vec<ExpnId>) { fragment.mut_visit_with(self); if let AstFragment::Items(mut items) = fragment { for derive in derives { - match self.remove(NodeId::placeholder_from_mark(derive)) { + match self.remove(NodeId::placeholder_from_expn_id(derive)) { AstFragment::Items(derived_items) => items.extend(derived_items), _ => unreachable!(), } diff --git a/src/libsyntax/ext/proc_macro.rs b/src/libsyntax/ext/proc_macro.rs new file mode 100644 index 00000000000..c17b6f6b424 --- /dev/null +++ b/src/libsyntax/ext/proc_macro.rs @@ -0,0 +1,217 @@ +use crate::ast::{self, ItemKind, Attribute, Mac}; +use crate::attr::{mark_used, mark_known}; +use crate::errors::{Applicability, FatalError}; +use crate::ext::base::{self, *}; +use crate::ext::proc_macro_server; +use crate::parse::{self, token}; +use crate::parse::parser::PathStyle; +use crate::symbol::sym; +use crate::tokenstream::{self, TokenStream}; +use crate::visit::Visitor; + +use rustc_data_structures::sync::Lrc; +use syntax_pos::{Span, DUMMY_SP}; + +const EXEC_STRATEGY: proc_macro::bridge::server::SameThread = + proc_macro::bridge::server::SameThread; + +pub struct BangProcMacro { + pub client: proc_macro::bridge::client::Client< + fn(proc_macro::TokenStream) -> proc_macro::TokenStream, + >, +} + +impl base::ProcMacro for BangProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + input: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc macro panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct AttrProcMacro { + pub client: proc_macro::bridge::client::Client< + fn(proc_macro::TokenStream, proc_macro::TokenStream) -> proc_macro::TokenStream, + >, +} + +impl base::AttrProcMacro for AttrProcMacro { + fn expand<'cx>(&self, + ecx: &'cx mut ExtCtxt<'_>, + span: Span, + annotation: TokenStream, + annotated: TokenStream) + -> TokenStream { + let server = proc_macro_server::Rustc::new(ecx); + match self.client.run(&EXEC_STRATEGY, server, annotation, annotated) { + Ok(stream) => stream, + Err(e) => { + let msg = "custom attribute panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + } + } +} + +pub struct ProcMacroDerive { + pub client: proc_macro::bridge::client::Client< + fn(proc_macro::TokenStream) -> proc_macro::TokenStream, + >, + pub attrs: Vec<ast::Name>, +} + +impl MultiItemModifier for ProcMacroDerive { + fn expand(&self, + ecx: &mut ExtCtxt<'_>, + span: Span, + _meta_item: &ast::MetaItem, + item: Annotatable) + -> Vec<Annotatable> { + let item = match item { + Annotatable::Item(item) => item, + Annotatable::ImplItem(_) | + Annotatable::TraitItem(_) | + Annotatable::ForeignItem(_) | + Annotatable::Stmt(_) | + Annotatable::Expr(_) => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + }; + match item.node { + ItemKind::Struct(..) | + ItemKind::Enum(..) | + ItemKind::Union(..) => {}, + _ => { + ecx.span_err(span, "proc-macro derives may only be \ + applied to a struct, enum, or union"); + return Vec::new() + } + } + + // Mark attributes as known, and used. + MarkAttrs(&self.attrs).visit_item(&item); + + let token = token::Interpolated(Lrc::new(token::NtItem(item))); + let input = tokenstream::TokenTree::token(token, DUMMY_SP).into(); + + let server = proc_macro_server::Rustc::new(ecx); + let stream = match self.client.run(&EXEC_STRATEGY, server, input) { + Ok(stream) => stream, + Err(e) => { + let msg = "proc-macro derive panicked"; + let mut err = ecx.struct_span_fatal(span, msg); + if let Some(s) = e.as_str() { + err.help(&format!("message: {}", s)); + } + + err.emit(); + FatalError.raise(); + } + }; + + let error_count_before = ecx.parse_sess.span_diagnostic.err_count(); + let msg = "proc-macro derive produced unparseable tokens"; + + let mut parser = parse::stream_to_parser(ecx.parse_sess, stream, Some("proc-macro derive")); + let mut items = vec![]; + + loop { + match parser.parse_item() { + Ok(None) => break, + Ok(Some(item)) => { + items.push(Annotatable::Item(item)) + } + Err(mut err) => { + // FIXME: handle this better + err.cancel(); + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + } + } + + + // fail if there have been errors emitted + if ecx.parse_sess.span_diagnostic.err_count() > error_count_before { + ecx.struct_span_fatal(span, msg).emit(); + FatalError.raise(); + } + + items + } +} + +struct MarkAttrs<'a>(&'a [ast::Name]); + +impl<'a> Visitor<'a> for MarkAttrs<'a> { + fn visit_attribute(&mut self, attr: &Attribute) { + if let Some(ident) = attr.ident() { + if self.0.contains(&ident.name) { + mark_used(attr); + mark_known(attr); + } + } + } + + fn visit_mac(&mut self, _mac: &Mac) {} +} + +pub fn is_proc_macro_attr(attr: &Attribute) -> bool { + [sym::proc_macro, sym::proc_macro_attribute, sym::proc_macro_derive] + .iter().any(|kind| attr.check_name(*kind)) +} + +crate fn collect_derives(cx: &mut ExtCtxt<'_>, attrs: &mut Vec<ast::Attribute>) -> Vec<ast::Path> { + let mut result = Vec::new(); + attrs.retain(|attr| { + if attr.path != sym::derive { + return true; + } + if !attr.is_meta_item_list() { + cx.struct_span_err(attr.span, "malformed `derive` attribute input") + .span_suggestion( + attr.span, + "missing traits to be derived", + "#[derive(Trait1, Trait2, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).emit(); + return false; + } + + match attr.parse_list(cx.parse_sess, + |parser| parser.parse_path_allowing_meta(PathStyle::Mod)) { + Ok(traits) => { + result.extend(traits); + true + } + Err(mut e) => { + e.emit(); + false + } + } + }); + result +} diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs new file mode 100644 index 00000000000..36621ce7775 --- /dev/null +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -0,0 +1,715 @@ +use crate::ast; +use crate::ext::base::ExtCtxt; +use crate::parse::{self, token, ParseSess}; +use crate::parse::lexer::comments; +use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; + +use errors::{Diagnostic, DiagnosticBuilder}; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, FileName, MultiSpan, Pos, SourceFile, Span}; +use syntax_pos::hygiene::{SyntaxContext, Transparency}; +use syntax_pos::symbol::{kw, sym, Symbol}; + +use proc_macro::{Delimiter, Level, LineColumn, Spacing}; +use proc_macro::bridge::{server, TokenTree}; +use std::{ascii, panic}; +use std::ops::Bound; + +trait FromInternal<T> { + fn from_internal(x: T) -> Self; +} + +trait ToInternal<T> { + fn to_internal(self) -> T; +} + +impl FromInternal<token::DelimToken> for Delimiter { + fn from_internal(delim: token::DelimToken) -> Delimiter { + match delim { + token::Paren => Delimiter::Parenthesis, + token::Brace => Delimiter::Brace, + token::Bracket => Delimiter::Bracket, + token::NoDelim => Delimiter::None, + } + } +} + +impl ToInternal<token::DelimToken> for Delimiter { + fn to_internal(self) -> token::DelimToken { + match self { + Delimiter::Parenthesis => token::Paren, + Delimiter::Brace => token::Brace, + Delimiter::Bracket => token::Bracket, + Delimiter::None => token::NoDelim, + } + } +} + +impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)> + for TokenTree<Group, Punct, Ident, Literal> +{ + fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>)) + -> Self { + use crate::parse::token::*; + + let joint = is_joint == Joint; + let Token { kind, span } = match tree { + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); + return TokenTree::Group(Group { + delimiter, + stream: tts.into(), + span, + }); + } + tokenstream::TokenTree::Token(token) => token, + }; + + macro_rules! tt { + ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( + TokenTree::$ty(self::$ty { + $($field $(: $value)*,)+ + span, + }) + ); + ($ty:ident::$method:ident($($value:expr),*)) => ( + TokenTree::$ty(self::$ty::$method($($value,)* span)) + ); + } + macro_rules! op { + ($a:expr) => { + tt!(Punct::new($a, joint)) + }; + ($a:expr, $b:expr) => {{ + stack.push(tt!(Punct::new($b, joint))); + tt!(Punct::new($a, true)) + }}; + ($a:expr, $b:expr, $c:expr) => {{ + stack.push(tt!(Punct::new($c, joint))); + stack.push(tt!(Punct::new($b, true))); + tt!(Punct::new($a, true)) + }}; + } + + match kind { + Eq => op!('='), + Lt => op!('<'), + Le => op!('<', '='), + EqEq => op!('=', '='), + Ne => op!('!', '='), + Ge => op!('>', '='), + Gt => op!('>'), + AndAnd => op!('&', '&'), + OrOr => op!('|', '|'), + Not => op!('!'), + Tilde => op!('~'), + BinOp(Plus) => op!('+'), + BinOp(Minus) => op!('-'), + BinOp(Star) => op!('*'), + BinOp(Slash) => op!('/'), + BinOp(Percent) => op!('%'), + BinOp(Caret) => op!('^'), + BinOp(And) => op!('&'), + BinOp(Or) => op!('|'), + BinOp(Shl) => op!('<', '<'), + BinOp(Shr) => op!('>', '>'), + BinOpEq(Plus) => op!('+', '='), + BinOpEq(Minus) => op!('-', '='), + BinOpEq(Star) => op!('*', '='), + BinOpEq(Slash) => op!('/', '='), + BinOpEq(Percent) => op!('%', '='), + BinOpEq(Caret) => op!('^', '='), + BinOpEq(And) => op!('&', '='), + BinOpEq(Or) => op!('|', '='), + BinOpEq(Shl) => op!('<', '<', '='), + BinOpEq(Shr) => op!('>', '>', '='), + At => op!('@'), + Dot => op!('.'), + DotDot => op!('.', '.'), + DotDotDot => op!('.', '.', '.'), + DotDotEq => op!('.', '.', '='), + Comma => op!(','), + Semi => op!(';'), + Colon => op!(':'), + ModSep => op!(':', ':'), + RArrow => op!('-', '>'), + LArrow => op!('<', '-'), + FatArrow => op!('=', '>'), + Pound => op!('#'), + Dollar => op!('$'), + Question => op!('?'), + SingleQuote => op!('\''), + + Ident(name, false) if name == kw::DollarCrate => tt!(Ident::dollar_crate()), + Ident(name, is_raw) => tt!(Ident::new(name, is_raw)), + Lifetime(name) => { + let ident = ast::Ident::new(name, span).without_first_quote(); + stack.push(tt!(Ident::new(ident.name, false))); + tt!(Punct::new('\'', true)) + } + Literal(lit) => tt!(Literal { lit }), + DocComment(c) => { + let style = comments::doc_comment_style(&c.as_str()); + let stripped = comments::strip_doc_comment_decoration(&c.as_str()); + let mut escaped = String::new(); + for ch in stripped.chars() { + escaped.extend(ch.escape_debug()); + } + let stream = vec![ + Ident(sym::doc, false), + Eq, + TokenKind::lit(token::Str, Symbol::intern(&escaped), None), + ] + .into_iter() + .map(|kind| tokenstream::TokenTree::token(kind, span)) + .collect(); + stack.push(TokenTree::Group(Group { + delimiter: Delimiter::Bracket, + stream, + span: DelimSpan::from_single(span), + })); + if style == ast::AttrStyle::Inner { + stack.push(tt!(Punct::new('!', false))); + } + tt!(Punct::new('#', false)) + } + + Interpolated(nt) => { + let stream = nt.to_tokenstream(sess, span); + TokenTree::Group(Group { + delimiter: Delimiter::None, + stream, + span: DelimSpan::from_single(span), + }) + } + + OpenDelim(..) | CloseDelim(..) => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), + } + } +} + +impl ToInternal<TokenStream> for TokenTree<Group, Punct, Ident, Literal> { + fn to_internal(self) -> TokenStream { + use crate::parse::token::*; + + let (ch, joint, span) = match self { + TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), + TokenTree::Group(Group { + delimiter, + stream, + span, + }) => { + return tokenstream::TokenTree::Delimited( + span, + delimiter.to_internal(), + stream.into(), + ) + .into(); + } + TokenTree::Ident(self::Ident { sym, is_raw, span }) => { + return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Integer, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let integer = TokenKind::lit(token::Integer, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(integer, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { + lit: token::Lit { kind: token::Float, symbol, suffix }, + span, + }) if symbol.as_str().starts_with("-") => { + let minus = BinOp(BinOpToken::Minus); + let symbol = Symbol::intern(&symbol.as_str()[1..]); + let float = TokenKind::lit(token::Float, symbol, suffix); + let a = tokenstream::TokenTree::token(minus, span); + let b = tokenstream::TokenTree::token(float, span); + return vec![a, b].into_iter().collect(); + } + TokenTree::Literal(self::Literal { lit, span }) => { + return tokenstream::TokenTree::token(Literal(lit), span).into() + } + }; + + let kind = match ch { + '=' => Eq, + '<' => Lt, + '>' => Gt, + '!' => Not, + '~' => Tilde, + '+' => BinOp(Plus), + '-' => BinOp(Minus), + '*' => BinOp(Star), + '/' => BinOp(Slash), + '%' => BinOp(Percent), + '^' => BinOp(Caret), + '&' => BinOp(And), + '|' => BinOp(Or), + '@' => At, + '.' => Dot, + ',' => Comma, + ';' => Semi, + ':' => Colon, + '#' => Pound, + '$' => Dollar, + '?' => Question, + '\'' => SingleQuote, + _ => unreachable!(), + }; + + let tree = tokenstream::TokenTree::token(kind, span); + TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + } +} + +impl ToInternal<errors::Level> for Level { + fn to_internal(self) -> errors::Level { + match self { + Level::Error => errors::Level::Error, + Level::Warning => errors::Level::Warning, + Level::Note => errors::Level::Note, + Level::Help => errors::Level::Help, + _ => unreachable!("unknown proc_macro::Level variant: {:?}", self), + } + } +} + +#[derive(Clone)] +pub struct TokenStreamIter { + cursor: tokenstream::Cursor, + stack: Vec<TokenTree<Group, Punct, Ident, Literal>>, +} + +#[derive(Clone)] +pub struct Group { + delimiter: Delimiter, + stream: TokenStream, + span: DelimSpan, +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Punct { + ch: char, + // NB. not using `Spacing` here because it doesn't implement `Hash`. + joint: bool, + span: Span, +} + +impl Punct { + fn new(ch: char, joint: bool, span: Span) -> Punct { + const LEGAL_CHARS: &[char] = &['=', '<', '>', '!', '~', '+', '-', '*', '/', '%', '^', + '&', '|', '@', '.', ',', ';', ':', '#', '$', '?', '\'']; + if !LEGAL_CHARS.contains(&ch) { + panic!("unsupported character `{:?}`", ch) + } + Punct { ch, joint, span } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, Hash)] +pub struct Ident { + sym: Symbol, + is_raw: bool, + span: Span, +} + +impl Ident { + fn is_valid(string: &str) -> bool { + let mut chars = string.chars(); + if let Some(start) = chars.next() { + (start == '_' || start.is_xid_start()) + && chars.all(|cont| cont == '_' || cont.is_xid_continue()) + } else { + false + } + } + fn new(sym: Symbol, is_raw: bool, span: Span) -> Ident { + let string = sym.as_str(); + if !Self::is_valid(&string) { + panic!("`{:?}` is not a valid identifier", string) + } + // Get rid of gensyms to conservatively check rawness on the string contents only. + if is_raw && !sym.as_interned_str().as_symbol().can_be_raw() { + panic!("`{}` cannot be a raw identifier", string); + } + Ident { sym, is_raw, span } + } + fn dollar_crate(span: Span) -> Ident { + // `$crate` is accepted as an ident only if it comes from the compiler. + Ident { sym: kw::DollarCrate, is_raw: false, span } + } +} + +// FIXME(eddyb) `Literal` should not expose internal `Debug` impls. +#[derive(Clone, Debug)] +pub struct Literal { + lit: token::Lit, + span: Span, +} + +pub(crate) struct Rustc<'a> { + sess: &'a ParseSess, + def_site: Span, + call_site: Span, +} + +impl<'a> Rustc<'a> { + pub fn new(cx: &'a ExtCtxt<'_>) -> Self { + // No way to determine def location for a proc macro right now, so use call location. + let location = cx.current_expansion.id.expn_info().unwrap().call_site; + let to_span = |transparency| { + location.with_ctxt( + SyntaxContext::empty() + .apply_mark_with_transparency(cx.current_expansion.id, transparency), + ) + }; + Rustc { + sess: cx.parse_sess, + def_site: to_span(Transparency::Opaque), + call_site: to_span(Transparency::Transparent), + } + } + + fn lit(&mut self, kind: token::LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Literal { + Literal { + lit: token::Lit::new(kind, symbol, suffix), + span: server::Span::call_site(self), + } + } +} + +impl server::Types for Rustc<'_> { + type TokenStream = TokenStream; + type TokenStreamBuilder = tokenstream::TokenStreamBuilder; + type TokenStreamIter = TokenStreamIter; + type Group = Group; + type Punct = Punct; + type Ident = Ident; + type Literal = Literal; + type SourceFile = Lrc<SourceFile>; + type MultiSpan = Vec<Span>; + type Diagnostic = Diagnostic; + type Span = Span; +} + +impl server::TokenStream for Rustc<'_> { + fn new(&mut self) -> Self::TokenStream { + TokenStream::empty() + } + fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { + stream.is_empty() + } + fn from_str(&mut self, src: &str) -> Self::TokenStream { + parse::parse_stream_from_source_str( + FileName::proc_macro_source_code(src), + src.to_string(), + self.sess, + Some(self.call_site), + ) + } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { + stream.to_string() + } + fn from_token_tree( + &mut self, + tree: TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>, + ) -> Self::TokenStream { + tree.to_internal() + } + fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter { + TokenStreamIter { + cursor: stream.trees(), + stack: vec![], + } + } +} + +impl server::TokenStreamBuilder for Rustc<'_> { + fn new(&mut self) -> Self::TokenStreamBuilder { + tokenstream::TokenStreamBuilder::new() + } + fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) { + builder.push(stream); + } + fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream { + builder.build() + } +} + +impl server::TokenStreamIter for Rustc<'_> { + fn next( + &mut self, + iter: &mut Self::TokenStreamIter, + ) -> Option<TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> { + loop { + let tree = iter.stack.pop().or_else(|| { + let next = iter.cursor.next_with_joint()?; + Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) + })?; + // HACK: The condition "dummy span + group with empty delimiter" represents an AST + // fragment approximately converted into a token stream. This may happen, for + // example, with inputs to proc macro attributes, including derives. Such "groups" + // need to flattened during iteration over stream's token trees. + // Eventually this needs to be removed in favor of keeping original token trees + // and not doing the roundtrip through AST. + if let TokenTree::Group(ref group) = tree { + if group.delimiter == Delimiter::None && group.span.entire().is_dummy() { + iter.cursor.append(group.stream.clone()); + continue; + } + } + return Some(tree); + } + } +} + +impl server::Group for Rustc<'_> { + fn new(&mut self, delimiter: Delimiter, stream: Self::TokenStream) -> Self::Group { + Group { + delimiter, + stream, + span: DelimSpan::from_single(server::Span::call_site(self)), + } + } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { + group.delimiter + } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { + group.stream.clone() + } + fn span(&mut self, group: &Self::Group) -> Self::Span { + group.span.entire() + } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { + group.span.open + } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { + group.span.close + } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { + group.span = DelimSpan::from_single(span); + } +} + +impl server::Punct for Rustc<'_> { + fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { + Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) + } + fn as_char(&mut self, punct: Self::Punct) -> char { + punct.ch + } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { + if punct.joint { + Spacing::Joint + } else { + Spacing::Alone + } + } + fn span(&mut self, punct: Self::Punct) -> Self::Span { + punct.span + } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { + Punct { span, ..punct } + } +} + +impl server::Ident for Rustc<'_> { + fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { + Ident::new(Symbol::intern(string), is_raw, span) + } + fn span(&mut self, ident: Self::Ident) -> Self::Span { + ident.span + } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { + Ident { span, ..ident } + } +} + +impl server::Literal for Rustc<'_> { + // FIXME(eddyb) `Literal` should not expose internal `Debug` impls. + fn debug(&mut self, literal: &Self::Literal) -> String { + format!("{:?}", literal) + } + fn integer(&mut self, n: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), None) + } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { + self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) + } + fn float(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), None) + } + fn f32(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) + } + fn f64(&mut self, n: &str) -> Self::Literal { + self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) + } + fn string(&mut self, string: &str) -> Self::Literal { + let mut escaped = String::new(); + for ch in string.chars() { + escaped.extend(ch.escape_debug()); + } + self.lit(token::Str, Symbol::intern(&escaped), None) + } + fn character(&mut self, ch: char) -> Self::Literal { + let mut escaped = String::new(); + escaped.extend(ch.escape_unicode()); + self.lit(token::Char, Symbol::intern(&escaped), None) + } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { + let string = bytes + .iter() + .cloned() + .flat_map(ascii::escape_default) + .map(Into::<char>::into) + .collect::<String>(); + self.lit(token::ByteStr, Symbol::intern(&string), None) + } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { + literal.span + } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { + literal.span = span; + } + fn subspan( + &mut self, + literal: &Self::Literal, + start: Bound<usize>, + end: Bound<usize>, + ) -> Option<Self::Span> { + let span = literal.span; + let length = span.hi().to_usize() - span.lo().to_usize(); + + let start = match start { + Bound::Included(lo) => lo, + Bound::Excluded(lo) => lo + 1, + Bound::Unbounded => 0, + }; + + let end = match end { + Bound::Included(hi) => hi + 1, + Bound::Excluded(hi) => hi, + Bound::Unbounded => length, + }; + + // Bounds check the values, preventing addition overflow and OOB spans. + if start > u32::max_value() as usize + || end > u32::max_value() as usize + || (u32::max_value() - start as u32) < span.lo().to_u32() + || (u32::max_value() - end as u32) < span.lo().to_u32() + || start >= end + || end > length + { + return None; + } + + let new_lo = span.lo() + BytePos::from_usize(start); + let new_hi = span.lo() + BytePos::from_usize(end); + Some(span.with_lo(new_lo).with_hi(new_hi)) + } +} + +impl server::SourceFile for Rustc<'_> { + fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { + Lrc::ptr_eq(file1, file2) + } + fn path(&mut self, file: &Self::SourceFile) -> String { + match file.name { + FileName::Real(ref path) => path + .to_str() + .expect("non-UTF8 file path in `proc_macro::SourceFile::path`") + .to_string(), + _ => file.name.to_string(), + } + } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { + file.is_real_file() + } +} + +impl server::MultiSpan for Rustc<'_> { + fn new(&mut self) -> Self::MultiSpan { + vec![] + } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { + spans.push(span) + } +} + +impl server::Diagnostic for Rustc<'_> { + fn new(&mut self, level: Level, msg: &str, spans: Self::MultiSpan) -> Self::Diagnostic { + let mut diag = Diagnostic::new(level.to_internal(), msg); + diag.set_span(MultiSpan::from_spans(spans)); + diag + } + fn sub( + &mut self, + diag: &mut Self::Diagnostic, + level: Level, + msg: &str, + spans: Self::MultiSpan, + ) { + diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); + } + fn emit(&mut self, diag: Self::Diagnostic) { + DiagnosticBuilder::new_diagnostic(&self.sess.span_diagnostic, diag).emit() + } +} + +impl server::Span for Rustc<'_> { + fn debug(&mut self, span: Self::Span) -> String { + format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) + } + fn def_site(&mut self) -> Self::Span { + self.def_site + } + fn call_site(&mut self) -> Self::Span { + self.call_site + } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { + self.sess.source_map().lookup_char_pos(span.lo()).file + } + fn parent(&mut self, span: Self::Span) -> Option<Self::Span> { + span.ctxt().outer_expn_info().map(|i| i.call_site) + } + fn source(&mut self, span: Self::Span) -> Self::Span { + span.source_callsite() + } + fn start(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.lo()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn end(&mut self, span: Self::Span) -> LineColumn { + let loc = self.sess.source_map().lookup_char_pos(span.hi()); + LineColumn { + line: loc.line, + column: loc.col.to_usize(), + } + } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option<Self::Span> { + let self_loc = self.sess.source_map().lookup_char_pos(first.lo()); + let other_loc = self.sess.source_map().lookup_char_pos(second.lo()); + + if self_loc.file.name != other_loc.file.name { + return None; + } + + Some(first.to(second)) + } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { + span.with_ctxt(at.ctxt()) + } + fn source_text(&mut self, span: Self::Span) -> Option<String> { + self.sess.source_map().span_to_snippet(span).ok() + } +} diff --git a/src/libsyntax/ext/source_util.rs b/src/libsyntax/ext/source_util.rs deleted file mode 100644 index 549de1628eb..00000000000 --- a/src/libsyntax/ext/source_util.rs +++ /dev/null @@ -1,197 +0,0 @@ -use crate::ast; -use crate::ext::base::{self, *}; -use crate::ext::build::AstBuilder; -use crate::parse::{self, token, DirectoryOwnership}; -use crate::print::pprust; -use crate::ptr::P; -use crate::symbol::Symbol; -use crate::tokenstream; - -use smallvec::SmallVec; -use syntax_pos::{self, Pos, Span, FileName}; - -use std::fs; -use std::io::ErrorKind; -use std::path::PathBuf; -use rustc_data_structures::sync::Lrc; - -// These macros all relate to the file system; they either return -// the column/row/filename of the expression, or they include -// a given file into the current one. - -/// line!(): expands to the current line number -pub fn expand_line(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - base::check_zero_tts(cx, sp, tts, "line!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - - base::MacEager::expr(cx.expr_u32(topmost, loc.line as u32)) -} - -/* column!(): expands to the current column number */ -pub fn expand_column(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - base::check_zero_tts(cx, sp, tts, "column!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - - base::MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1)) -} - -/* __rust_unstable_column!(): expands to the current column number */ -pub fn expand_column_gated(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - if sp.allows_unstable("__rust_unstable_column") { - expand_column(cx, sp, tts) - } else { - cx.span_fatal(sp, "the __rust_unstable_column macro is unstable"); - } -} - -/// file!(): expands to the current filename */ -/// The source_file (`loc.file`) contains a bunch more information we could spit -/// out if we wanted. -pub fn expand_file(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - base::check_zero_tts(cx, sp, tts, "file!"); - - let topmost = cx.expansion_cause().unwrap_or(sp); - let loc = cx.source_map().lookup_char_pos(topmost.lo()); - base::MacEager::expr(cx.expr_str(topmost, Symbol::intern(&loc.file.name.to_string()))) -} - -pub fn expand_stringify(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - let s = pprust::tts_to_string(tts); - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&s))) -} - -pub fn expand_mod(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - base::check_zero_tts(cx, sp, tts, "module_path!"); - let mod_path = &cx.current_expansion.module.mod_path; - let string = mod_path.iter().map(|x| x.to_string()).collect::<Vec<String>>().join("::"); - - base::MacEager::expr(cx.expr_str(sp, Symbol::intern(&string))) -} - -/// include! : parse the given file as an expr -/// This is generally a bad idea because it's going to behave -/// unhygienically. -pub fn expand_include<'cx>(cx: &'cx mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'cx> { - let file = match get_single_str_from_tts(cx, sp, tts, "include!") { - Some(f) => f, - None => return DummyResult::any(sp), - }; - // The file will be added to the code map by the parser - let path = res_rel_file(cx, sp, file); - let directory_ownership = DirectoryOwnership::Owned { relative: None }; - let p = parse::new_sub_parser_from_file(cx.parse_sess(), &path, directory_ownership, None, sp); - - struct ExpandResult<'a> { - p: parse::parser::Parser<'a>, - } - impl<'a> base::MacResult for ExpandResult<'a> { - fn make_expr(mut self: Box<ExpandResult<'a>>) -> Option<P<ast::Expr>> { - Some(panictry!(self.p.parse_expr())) - } - - fn make_items(mut self: Box<ExpandResult<'a>>) -> Option<SmallVec<[P<ast::Item>; 1]>> { - let mut ret = SmallVec::new(); - while self.p.token != token::Eof { - match panictry!(self.p.parse_item()) { - Some(item) => ret.push(item), - None => self.p.diagnostic().span_fatal(self.p.span, - &format!("expected item, found `{}`", - self.p.this_token_to_string())) - .raise() - } - } - Some(ret) - } - } - - Box::new(ExpandResult { p }) -} - -// include_str! : read the given file, insert it as a literal string expr -pub fn expand_include_str(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - let file = match get_single_str_from_tts(cx, sp, tts, "include_str!") { - Some(f) => f, - None => return DummyResult::expr(sp) - }; - let file = res_rel_file(cx, sp, file); - match fs::read_to_string(&file) { - Ok(src) => { - let interned_src = Symbol::intern(&src); - - // Add this input file to the code map to make it available as - // dependency information - cx.source_map().new_source_file(file.into(), src); - - base::MacEager::expr(cx.expr_str(sp, interned_src)) - }, - Err(ref e) if e.kind() == ErrorKind::InvalidData => { - cx.span_err(sp, &format!("{} wasn't a utf-8 file", file.display())); - DummyResult::expr(sp) - } - Err(e) => { - cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); - DummyResult::expr(sp) - } - } -} - -pub fn expand_include_bytes(cx: &mut ExtCtxt<'_>, sp: Span, tts: &[tokenstream::TokenTree]) - -> Box<dyn base::MacResult+'static> { - let file = match get_single_str_from_tts(cx, sp, tts, "include_bytes!") { - Some(f) => f, - None => return DummyResult::expr(sp) - }; - let file = res_rel_file(cx, sp, file); - match fs::read(&file) { - Ok(bytes) => { - // Add the contents to the source map if it contains UTF-8. - let (contents, bytes) = match String::from_utf8(bytes) { - Ok(s) => { - let bytes = s.as_bytes().to_owned(); - (s, bytes) - }, - Err(e) => (String::new(), e.into_bytes()), - }; - cx.source_map().new_source_file(file.into(), contents); - - base::MacEager::expr(cx.expr_lit(sp, ast::LitKind::ByteStr(Lrc::new(bytes)))) - }, - Err(e) => { - cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e)); - DummyResult::expr(sp) - } - } -} - -// resolve a file-system path to an absolute file-system path (if it -// isn't already) -fn res_rel_file(cx: &mut ExtCtxt<'_>, sp: syntax_pos::Span, arg: String) -> PathBuf { - let arg = PathBuf::from(arg); - // Relative paths are resolved relative to the file in which they are found - // after macro expansion (that is, they are unhygienic). - if !arg.is_absolute() { - let callsite = sp.source_callsite(); - let mut path = match cx.source_map().span_to_unmapped_path(callsite) { - FileName::Real(path) => path, - FileName::DocTest(path, _) => path, - other => panic!("cannot resolve relative path in non-file source `{}`", other), - }; - path.pop(); - path.push(arg); - path - } else { - arg - } -} diff --git a/src/libsyntax/ext/tt/macro_check.rs b/src/libsyntax/ext/tt/macro_check.rs new file mode 100644 index 00000000000..5af97199902 --- /dev/null +++ b/src/libsyntax/ext/tt/macro_check.rs @@ -0,0 +1,626 @@ +//! Checks that meta-variables in macro definition are correctly declared and used. +//! +//! # What is checked +//! +//! ## Meta-variables must not be bound twice +//! +//! ``` +//! macro_rules! foo { ($x:tt $x:tt) => { $x }; } +//! ``` +//! +//! This check is sound (no false-negative) and complete (no false-positive). +//! +//! ## Meta-variables must not be free +//! +//! ``` +//! macro_rules! foo { () => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat at least as many times as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)*) => { $x }; } +//! ``` +//! +//! This check is also done at macro instantiation but only if the branch is taken. +//! +//! ## Meta-variables must repeat with the same Kleene operators as their binder +//! +//! ``` +//! macro_rules! foo { ($($x:tt)+) => { $($x)* }; } +//! ``` +//! +//! This check is not done at macro instantiation. +//! +//! # Disclaimer +//! +//! In the presence of nested macros (a macro defined in a macro), those checks may have false +//! positives and false negatives. We try to detect those cases by recognizing potential macro +//! definitions in RHSes, but nested macros may be hidden through the use of particular values of +//! meta-variables. +//! +//! ## Examples of false positive +//! +//! False positives can come from cases where we don't recognize a nested macro, because it depends +//! on particular values of meta-variables. In the following example, we think both instances of +//! `$x` are free, which is a correct statement if `$name` is anything but `macro_rules`. But when +//! `$name` is `macro_rules`, like in the instantiation below, then `$x:tt` is actually a binder of +//! the nested macro and `$x` is bound to it. +//! +//! ``` +//! macro_rules! foo { ($name:ident) => { $name! bar { ($x:tt) => { $x }; } }; } +//! foo!(macro_rules); +//! ``` +//! +//! False positives can also come from cases where we think there is a nested macro while there +//! isn't. In the following example, we think `$x` is free, which is incorrect because `bar` is not +//! a nested macro since it is not evaluated as code by `stringify!`. +//! +//! ``` +//! macro_rules! foo { () => { stringify!(macro_rules! bar { () => { $x }; }) }; } +//! ``` +//! +//! ## Examples of false negative +//! +//! False negatives can come from cases where we don't recognize a meta-variable, because it depends +//! on particular values of meta-variables. In the following examples, we don't see that if `$d` is +//! instantiated with `$` then `$d z` becomes `$z` in the nested macro definition and is thus a free +//! meta-variable. Note however, that if `foo` is instantiated, then we would check the definition +//! of `bar` and would see the issue. +//! +//! ``` +//! macro_rules! foo { ($d:tt) => { macro_rules! bar { ($y:tt) => { $d z }; } }; } +//! ``` +//! +//! # How it is checked +//! +//! There are 3 main functions: `check_binders`, `check_occurrences`, and `check_nested_macro`. They +//! all need some kind of environment. +//! +//! ## Environments +//! +//! Environments are used to pass information. +//! +//! ### From LHS to RHS +//! +//! When checking a LHS with `check_binders`, we produce (and use) an environment for binders, +//! namely `Binders`. This is a mapping from binder name to information about that binder: the span +//! of the binder for error messages and the stack of Kleene operators under which it was bound in +//! the LHS. +//! +//! This environment is used by both the LHS and RHS. The LHS uses it to detect duplicate binders. +//! The RHS uses it to detect the other errors. +//! +//! ### From outer macro to inner macro +//! +//! When checking the RHS of an outer macro and we detect a nested macro definition, we push the +//! current state, namely `MacroState`, to an environment of nested macro definitions. Each state +//! stores the LHS binders when entering the macro definition as well as the stack of Kleene +//! operators under which the inner macro is defined in the RHS. +//! +//! This environment is a stack representing the nesting of macro definitions. As such, the stack of +//! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks +//! stored when entering a macro definition starting from the state in which the meta-variable is +//! bound. +use crate::ast::NodeId; +use crate::early_buffered_lints::BufferedEarlyLintId; +use crate::ext::tt::quoted::{KleeneToken, TokenTree}; +use crate::parse::token::TokenKind; +use crate::parse::token::{DelimToken, Token}; +use crate::parse::ParseSess; +use crate::symbol::{kw, sym}; + +use rustc_data_structures::fx::FxHashMap; +use smallvec::SmallVec; +use syntax_pos::{symbol::Ident, MultiSpan, Span}; + +/// Stack represented as linked list. +/// +/// Those are used for environments because they grow incrementally and are not mutable. +enum Stack<'a, T> { + /// Empty stack. + Empty, + /// A non-empty stack. + Push { + /// The top element. + top: T, + /// The previous elements. + prev: &'a Stack<'a, T>, + }, +} + +impl<'a, T> Stack<'a, T> { + /// Returns whether a stack is empty. + fn is_empty(&self) -> bool { + match *self { + Stack::Empty => true, + _ => false, + } + } + + /// Returns a new stack with an element of top. + fn push(&'a self, top: T) -> Stack<'a, T> { + Stack::Push { top, prev: self } + } +} + +impl<'a, T> Iterator for &'a Stack<'a, T> { + type Item = &'a T; + + // Iterates from top to bottom of the stack. + fn next(&mut self) -> Option<&'a T> { + match *self { + Stack::Empty => None, + Stack::Push { ref top, ref prev } => { + *self = prev; + Some(top) + } + } + } +} + +impl From<&Stack<'_, KleeneToken>> for SmallVec<[KleeneToken; 1]> { + fn from(ops: &Stack<'_, KleeneToken>) -> SmallVec<[KleeneToken; 1]> { + let mut ops: SmallVec<[KleeneToken; 1]> = ops.cloned().collect(); + // The stack is innermost on top. We want outermost first. + ops.reverse(); + ops + } +} + +/// Information attached to a meta-variable binder in LHS. +struct BinderInfo { + /// The span of the meta-variable in LHS. + span: Span, + /// The stack of Kleene operators (outermost first). + ops: SmallVec<[KleeneToken; 1]>, +} + +/// An environment of meta-variables to their binder information. +type Binders = FxHashMap<Ident, BinderInfo>; + +/// The state at which we entered a macro definition in the RHS of another macro definition. +struct MacroState<'a> { + /// The binders of the branch where we entered the macro definition. + binders: &'a Binders, + /// The stack of Kleene operators (outermost first) where we entered the macro definition. + ops: SmallVec<[KleeneToken; 1]>, +} + +/// Checks that meta-variables are used correctly in a macro definition. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is used when no spans are available +/// - `lhses` and `rhses` should have the same length and represent the macro definition +pub fn check_meta_variables( + sess: &ParseSess, + node_id: NodeId, + span: Span, + lhses: &[TokenTree], + rhses: &[TokenTree], +) -> bool { + if lhses.len() != rhses.len() { + sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes") + } + let mut valid = true; + for (lhs, rhs) in lhses.iter().zip(rhses.iter()) { + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, &Stack::Empty, &mut binders, &Stack::Empty, &mut valid); + check_occurrences(sess, node_id, rhs, &Stack::Empty, &binders, &Stack::Empty, &mut valid); + } + valid +} + +/// Checks `lhs` as part of the LHS of a macro definition, extends `binders` with new binders, and +/// sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `lhs` is checked as part of a LHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the LHS +/// - `ops` is the stack of Kleene operators from the LHS +/// - `valid` is set in case of errors +fn check_binders( + sess: &ParseSess, + node_id: NodeId, + lhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &mut Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *lhs { + TokenTree::Token(..) => {} + // This can only happen when checking a nested macro because this LHS is then in the RHS of + // the outer macro. See ui/macros/macro-of-higher-order.rs where $y:$fragment in the + // LHS of the nested macro (and RHS of the outer macro) is parsed as MetaVar(y) Colon + // MetaVar(fragment) and not as MetaVarDecl(y, fragment). + TokenTree::MetaVar(span, name) => { + if macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs"); + } + // There are 3 possibilities: + if let Some(prev_info) = binders.get(&name) { + // 1. The meta-variable is already bound in the current LHS: This is an error. + let mut span = MultiSpan::from_span(span); + span.push_span_label(prev_info.span, "previous declaration".into()); + buffer_lint(sess, span, node_id, "duplicate matcher binding"); + } else if get_binder_info(macros, binders, name).is_none() { + // 2. The meta-variable is free: This is a binder. + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } else { + // 3. The meta-variable is bound: This is an occurrence. + check_occurrences(sess, node_id, lhs, macros, binders, ops, valid); + } + } + // Similarly, this can only happen when checking a toplevel macro. + TokenTree::MetaVarDecl(span, name, _kind) => { + if !macros.is_empty() { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs"); + } + if let Some(prev_info) = get_binder_info(macros, binders, name) { + // Duplicate binders at the top-level macro definition are errors. The lint is only + // for nested macro definitions. + sess.span_diagnostic + .struct_span_err(span, "duplicate matcher binding") + .span_note(prev_info.span, "previous declaration was here") + .emit(); + *valid = false; + } else { + binders.insert(name, BinderInfo { span, ops: ops.into() }); + } + } + TokenTree::Delimited(_, ref del) => { + for tt in &del.tts { + check_binders(sess, node_id, tt, macros, binders, ops, valid); + } + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + for tt in &seq.tts { + check_binders(sess, node_id, tt, macros, binders, &ops, valid); + } + } + } +} + +/// Returns the binder information of a meta-variable. +/// +/// Arguments: +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the current binders +/// - `name` is the name of the meta-variable we are looking for +fn get_binder_info<'a>( + mut macros: &'a Stack<'a, MacroState<'a>>, + binders: &'a Binders, + name: Ident, +) -> Option<&'a BinderInfo> { + binders.get(&name).or_else(|| macros.find_map(|state| state.binders.get(&name))) +} + +/// Checks `rhs` as part of the RHS of a macro definition and sets `valid` to false in case of +/// errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `rhs` is checked as part of a RHS +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_occurrences( + sess: &ParseSess, + node_id: NodeId, + rhs: &TokenTree, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + match *rhs { + TokenTree::Token(..) => {} + TokenTree::MetaVarDecl(span, _name, _kind) => { + sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs") + } + TokenTree::MetaVar(span, name) => { + check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name); + } + TokenTree::Delimited(_, ref del) => { + check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid); + } + TokenTree::Sequence(_, ref seq) => { + let ops = ops.push(seq.kleene); + check_nested_occurrences(sess, node_id, &seq.tts, macros, binders, &ops, valid); + } + } +} + +/// Represents the processed prefix of a nested macro. +#[derive(Clone, Copy, PartialEq, Eq)] +enum NestedMacroState { + /// Nothing that matches a nested macro definition was processed yet. + Empty, + /// The token `macro_rules` was processed. + MacroRules, + /// The tokens `macro_rules!` were processed. + MacroRulesNot, + /// The tokens `macro_rules!` followed by a name were processed. The name may be either directly + /// an identifier or a meta-variable (that hopefully would be instantiated by an identifier). + MacroRulesNotName, + /// The keyword `macro` was processed. + Macro, + /// The keyword `macro` followed by a name was processed. + MacroName, + /// The keyword `macro` followed by a name and a token delimited by parentheses was processed. + MacroNameParen, +} + +/// Checks `tts` as part of the RHS of a macro definition, tries to recognize nested macro +/// definitions, and sets `valid` to false in case of errors. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `tts` is checked as part of a RHS and may contain macro definitions +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `valid` is set in case of errors +fn check_nested_occurrences( + sess: &ParseSess, + node_id: NodeId, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + valid: &mut bool, +) { + let mut state = NestedMacroState::Empty; + let nested_macros = macros.push(MacroState { binders, ops: ops.into() }); + let mut nested_binders = Binders::default(); + for tt in tts { + match (state, tt) { + ( + NestedMacroState::Empty, + &TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }), + ) => { + if name == sym::macro_rules { + state = NestedMacroState::MacroRules; + } else if name == kw::Macro { + state = NestedMacroState::Macro; + } + } + ( + NestedMacroState::MacroRules, + &TokenTree::Token(Token { kind: TokenKind::Not, .. }), + ) => { + state = NestedMacroState::MacroRulesNot; + } + ( + NestedMacroState::MacroRulesNot, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroRulesNotName; + } + (NestedMacroState::MacroRulesNot, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroRulesNotName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del)) + | (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + let legacy = state == NestedMacroState::MacroRulesNotName; + state = NestedMacroState::Empty; + let rest = + check_nested_macro(sess, node_id, legacy, &del.tts, &nested_macros, valid); + // If we did not check the whole macro definition, then check the rest as if outside + // the macro definition. + check_nested_occurrences( + sess, + node_id, + &del.tts[rest..], + macros, + binders, + ops, + valid, + ); + } + ( + NestedMacroState::Macro, + &TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }), + ) => { + state = NestedMacroState::MacroName; + } + (NestedMacroState::Macro, &TokenTree::MetaVar(..)) => { + state = NestedMacroState::MacroName; + // We check that the meta-variable is correctly used. + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Paren => + { + state = NestedMacroState::MacroNameParen; + nested_binders = Binders::default(); + check_binders( + sess, + node_id, + tt, + &nested_macros, + &mut nested_binders, + &Stack::Empty, + valid, + ); + } + (NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del)) + if del.delim == DelimToken::Brace => + { + state = NestedMacroState::Empty; + check_occurrences( + sess, + node_id, + tt, + &nested_macros, + &nested_binders, + &Stack::Empty, + valid, + ); + } + (_, ref tt) => { + state = NestedMacroState::Empty; + check_occurrences(sess, node_id, tt, macros, binders, ops, valid); + } + } + } +} + +/// Checks the body of nested macro, returns where the check stopped, and sets `valid` to false in +/// case of errors. +/// +/// The token trees are checked as long as they look like a list of (LHS) => {RHS} token trees. This +/// check is a best-effort to detect a macro definition. It returns the position in `tts` where we +/// stopped checking because we detected we were not in a macro definition anymore. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `legacy` specifies whether the macro is legacy +/// - `tts` is checked as a list of (LHS) => {RHS} +/// - `macros` is the stack of outer macros +/// - `valid` is set in case of errors +fn check_nested_macro( + sess: &ParseSess, + node_id: NodeId, + legacy: bool, + tts: &[TokenTree], + macros: &Stack<'_, MacroState<'_>>, + valid: &mut bool, +) -> usize { + let n = tts.len(); + let mut i = 0; + let separator = if legacy { TokenKind::Semi } else { TokenKind::Comma }; + loop { + // We expect 3 token trees: `(LHS) => {RHS}`. The separator is checked after. + if i + 2 >= n + || !tts[i].is_delimited() + || !tts[i + 1].is_token(&TokenKind::FatArrow) + || !tts[i + 2].is_delimited() + { + break; + } + let lhs = &tts[i]; + let rhs = &tts[i + 2]; + let mut binders = Binders::default(); + check_binders(sess, node_id, lhs, macros, &mut binders, &Stack::Empty, valid); + check_occurrences(sess, node_id, rhs, macros, &binders, &Stack::Empty, valid); + // Since the last semicolon is optional for legacy macros and decl_macro are not terminated, + // we increment our checked position by how many token trees we already checked (the 3 + // above) before checking for the separator. + i += 3; + if i == n || !tts[i].is_token(&separator) { + break; + } + // We increment our checked position for the semicolon. + i += 1; + } + i +} + +/// Checks that a meta-variable occurrence is valid. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `macros` is the stack of possible outer macros +/// - `binders` contains the binders of the associated LHS +/// - `ops` is the stack of Kleene operators from the RHS +/// - `span` is the span of the meta-variable to check +/// - `name` is the name of the meta-variable to check +fn check_ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + macros: &Stack<'_, MacroState<'_>>, + binders: &Binders, + ops: &Stack<'_, KleeneToken>, + span: Span, + name: Ident, +) { + let macros = macros.push(MacroState { binders, ops: ops.into() }); + // Accumulates the stacks the operators of each state until (and including when) the + // meta-variable is found. The innermost stack is first. + let mut acc: SmallVec<[&SmallVec<[KleeneToken; 1]>; 1]> = SmallVec::new(); + for state in ¯os { + acc.push(&state.ops); + if let Some(binder) = state.binders.get(&name) { + // This variable concatenates the stack of operators from the RHS of the LHS where the + // meta-variable was defined to where it is used (in possibly nested macros). The + // outermost operator is first. + let mut occurrence_ops: SmallVec<[KleeneToken; 2]> = SmallVec::new(); + // We need to iterate from the end to start with outermost stack. + for ops in acc.iter().rev() { + occurrence_ops.extend_from_slice(ops); + } + ops_is_prefix(sess, node_id, span, name, &binder.ops, &occurrence_ops); + return; + } + } + buffer_lint(sess, span.into(), node_id, &format!("unknown macro variable `{}`", name)); +} + +/// Returns whether `binder_ops` is a prefix of `occurrence_ops`. +/// +/// The stack of Kleene operators of a meta-variable occurrence just needs to have the stack of +/// Kleene operators of its binder as a prefix. +/// +/// Consider $i in the following example: +/// +/// ( $( $i:ident = $($j:ident),+ );* ) => { $($( $i += $j; )+)* } +/// +/// It occurs under the Kleene stack ["*", "+"] and is bound under ["*"] only. +/// +/// Arguments: +/// - `sess` is used to emit diagnostics and lints +/// - `node_id` is used to emit lints +/// - `span` is the span of the meta-variable being check +/// - `name` is the name of the meta-variable being check +/// - `binder_ops` is the stack of Kleene operators for the binder +/// - `occurrence_ops` is the stack of Kleene operators for the occurrence +fn ops_is_prefix( + sess: &ParseSess, + node_id: NodeId, + span: Span, + name: Ident, + binder_ops: &[KleeneToken], + occurrence_ops: &[KleeneToken], +) { + for (i, binder) in binder_ops.iter().enumerate() { + if i >= occurrence_ops.len() { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + let message = &format!("variable '{}' is still repeating at this depth", name); + buffer_lint(sess, span, node_id, message); + return; + } + let occurrence = &occurrence_ops[i]; + if occurrence.op != binder.op { + let mut span = MultiSpan::from_span(span); + span.push_span_label(binder.span, "expected repetition".into()); + span.push_span_label(occurrence.span, "conflicting repetition".into()); + let message = "meta-variable repeats with different Kleene operator"; + buffer_lint(sess, span, node_id, message); + return; + } + } +} + +fn buffer_lint(sess: &ParseSess, span: MultiSpan, node_id: NodeId, message: &str) { + sess.buffer_lint(BufferedEarlyLintId::MetaVariableMisuse, span, node_id, message); +} diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 1a419e7fada..dbf14daa30e 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -74,13 +74,13 @@ pub use NamedMatch::*; pub use ParseResult::*; use TokenTreeOrTokenTreeSlice::*; -use crate::ast::Ident; +use crate::ast::{Ident, Name}; use crate::ext::tt::quoted::{self, TokenTree}; use crate::parse::{Directory, ParseSess}; use crate::parse::parser::{Parser, PathStyle}; use crate::parse::token::{self, DocComment, Nonterminal, Token}; use crate::print::pprust; -use crate::symbol::keywords; +use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{DelimSpan, TokenStream}; use errors::FatalError; @@ -92,7 +92,6 @@ use rustc_data_structures::sync::Lrc; use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::mem; use std::ops::{Deref, DerefMut}; -use std::rc::Rc; // To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. @@ -156,7 +155,7 @@ type NamedMatchVec = SmallVec<[NamedMatch; 4]>; /// all the elements in that `SmallVec` strictly outlive the root stack slot /// lifetime. By separating `'tt` from `'root`, we can show that. #[derive(Clone)] -struct MatcherPos<'root, 'tt: 'root> { +struct MatcherPos<'root, 'tt> { /// The token or sequence of tokens that make up the matcher top_elts: TokenTreeOrTokenTreeSlice<'tt>, @@ -233,7 +232,7 @@ impl<'root, 'tt> MatcherPos<'root, 'tt> { // Therefore, the initial MatcherPos is always allocated on the stack, // subsequent ones (of which there aren't that many) are allocated on the heap, // and this type is used to encapsulate both cases. -enum MatcherPosHandle<'root, 'tt: 'root> { +enum MatcherPosHandle<'root, 'tt> { Ref(&'root mut MatcherPos<'root, 'tt>), Box(Box<MatcherPos<'root, 'tt>>), } @@ -273,14 +272,14 @@ pub enum ParseResult<T> { Success(T), /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. - Failure(syntax_pos::Span, Token, &'static str), + Failure(Token, &'static str), /// Fatal error (malformed macro?). Abort compilation. Error(syntax_pos::Span, String), } /// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es. /// This represents the mapping of metavars to the token trees they bind to. -pub type NamedParseResult = ParseResult<FxHashMap<Ident, Rc<NamedMatch>>>; +pub type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>; /// Count how many metavars are named in the given matcher `ms`. pub fn count_names(ms: &[TokenTree]) -> usize { @@ -373,7 +372,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>( sess: &ParseSess, m: &TokenTree, res: &mut I, - ret_val: &mut FxHashMap<Ident, Rc<NamedMatch>>, + ret_val: &mut FxHashMap<Ident, NamedMatch>, ) -> Result<(), (syntax_pos::Span, String)> { match *m { TokenTree::Sequence(_, ref seq) => for next_m in &seq.tts { @@ -382,7 +381,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>( TokenTree::Delimited(_, ref delim) => for next_m in &delim.tts { n_rec(sess, next_m, res.by_ref(), ret_val)?; }, - TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => { + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { return Err((span, "missing fragment specifier".to_string())); } @@ -390,8 +389,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>( TokenTree::MetaVarDecl(sp, bind_name, _) => { match ret_val.entry(bind_name) { Vacant(spot) => { - // FIXME(simulacrum): Don't construct Rc here - spot.insert(Rc::new(res.next().unwrap())); + spot.insert(res.next().unwrap()); } Occupied(..) => { return Err((sp, format!("duplicated bind name: {}", bind_name))) @@ -417,24 +415,24 @@ fn nameize<I: Iterator<Item = NamedMatch>>( /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For /// other tokens, this is "unexpected token...". -pub fn parse_failure_msg(tok: Token) -> String { - match tok { +pub fn parse_failure_msg(tok: &Token) -> String { + match tok.kind { token::Eof => "unexpected end of macro invocation".to_string(), _ => format!( "no rules expected the token `{}`", - pprust::token_to_string(&tok) + pprust::token_to_string(tok) ), } } /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) fn token_name_eq(t1: &Token, t2: &Token) -> bool { - if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) { - id1.name == id2.name && is_raw1 == is_raw2 - } else if let (Some(id1), Some(id2)) = (t1.lifetime(), t2.lifetime()) { - id1.name == id2.name + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name } else { - *t1 == *t2 + t1.kind == t2.kind } } @@ -467,7 +465,6 @@ fn inner_parse_loop<'root, 'tt>( eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>, token: &Token, - span: syntax_pos::Span, ) -> ParseResult<()> { // Pop items from `cur_items` until it is empty. while let Some(mut item) = cur_items.pop() { @@ -510,7 +507,7 @@ fn inner_parse_loop<'root, 'tt>( // Add matches from this repetition to the `matches` of `up` for idx in item.match_lo..item.match_hi { let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_open, span); + let span = DelimSpan::from_pair(item.sp_open, token.span); new_pos.push_match(idx, MatchedSeq(sub, span)); } @@ -554,9 +551,12 @@ fn inner_parse_loop<'root, 'tt>( match item.top_elts.get_tt(idx) { // Need to descend into a sequence TokenTree::Sequence(sp, seq) => { - // Examine the case where there are 0 matches of this sequence - if seq.op == quoted::KleeneOp::ZeroOrMore - || seq.op == quoted::KleeneOp::ZeroOrOne + // Examine the case where there are 0 matches of this sequence. We are + // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will + // result in a "no rules expected token" error by virtue of this matcher not + // working. + if seq.kleene.op == quoted::KleeneOp::ZeroOrMore + || seq.kleene.op == quoted::KleeneOp::ZeroOrOne { let mut new_item = item.clone(); new_item.match_cur += seq.num_captures; @@ -571,7 +571,7 @@ fn inner_parse_loop<'root, 'tt>( cur_items.push(MatcherPosHandle::Box(Box::new(MatcherPos { stack: smallvec![], sep: seq.separator.clone(), - seq_op: Some(seq.op), + seq_op: Some(seq.kleene.op), idx: 0, matches, match_lo: item.match_cur, @@ -584,7 +584,7 @@ fn inner_parse_loop<'root, 'tt>( } // We need to match a metavar (but the identifier is invalid)... this is an error - TokenTree::MetaVarDecl(span, _, id) if id.name == keywords::Invalid.name() => { + TokenTree::MetaVarDecl(span, _, id) if id.name == kw::Invalid => { if sess.missing_fragment_specifiers.borrow_mut().remove(&span) { return Error(span, "missing fragment specifier".to_string()); } @@ -595,7 +595,7 @@ fn inner_parse_loop<'root, 'tt>( TokenTree::MetaVarDecl(_, _, id) => { // Built-in nonterminals never start with these tokens, // so we can eliminate them from consideration. - if may_begin_with(&*id.as_str(), token) { + if may_begin_with(token, id.name) { bb_items.push(item); } } @@ -606,7 +606,8 @@ fn inner_parse_loop<'root, 'tt>( // // At the beginning of the loop, if we reach the end of the delimited submatcher, // we pop the stack to backtrack out of the descent. - seq @ TokenTree::Delimited(..) | seq @ TokenTree::Token(_, DocComment(..)) => { + seq @ TokenTree::Delimited(..) | + seq @ TokenTree::Token(Token { kind: DocComment(..), .. }) => { let lower_elts = mem::replace(&mut item.top_elts, Tt(seq)); let idx = item.idx; item.stack.push(MatcherTtFrame { @@ -618,7 +619,7 @@ fn inner_parse_loop<'root, 'tt>( } // We just matched a normal token. We can just advance the parser. - TokenTree::Token(_, ref t) if token_name_eq(t, token) => { + TokenTree::Token(t) if token_name_eq(&t, token) => { item.idx += 1; next_items.push(item); } @@ -655,7 +656,14 @@ pub fn parse( recurse_into_modules: bool, ) -> NamedParseResult { // Create a parser that can be used for the "black box" parts. - let mut parser = Parser::new(sess, tts, directory, recurse_into_modules, true); + let mut parser = Parser::new( + sess, + tts, + directory, + recurse_into_modules, + true, + crate::MACRO_ARGUMENTS, + ); // A queue of possible matcher positions. We initialize it with the matcher position in which // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then @@ -665,7 +673,7 @@ pub fn parse( // // This MatcherPos instance is allocated on the stack. All others -- and // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms, parser.span); + let mut initial = initial_matcher_pos(ms, parser.token.span); let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; let mut next_items = Vec::new(); @@ -687,10 +695,9 @@ pub fn parse( &mut eof_items, &mut bb_items, &parser.token, - parser.span, ) { Success(_) => {} - Failure(sp, tok, t) => return Failure(sp, tok, t), + Failure(token, msg) => return Failure(token, msg), Error(sp, msg) => return Error(sp, msg), } @@ -703,7 +710,7 @@ pub fn parse( // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, // either the parse is ambiguous (which should never happen) or there is a syntax error. - if token_name_eq(&parser.token, &token::Eof) { + if parser.token == token::Eof { if eof_items.len() == 1 { let matches = eof_items[0] .matches @@ -712,17 +719,16 @@ pub fn parse( return nameize(sess, ms, matches); } else if eof_items.len() > 1 { return Error( - parser.span, + parser.token.span, "ambiguity: multiple successful parses".to_string(), ); } else { return Failure( - if parser.span.is_dummy() { - parser.span + Token::new(token::Eof, if parser.token.span.is_dummy() { + parser.token.span } else { - sess.source_map().next_point(parser.span) - }, - token::Eof, + sess.source_map().next_point(parser.token.span) + }), "missing tokens in macro arguments", ); } @@ -745,7 +751,7 @@ pub fn parse( .join(" or "); return Error( - parser.span, + parser.token.span, format!( "local ambiguity: multiple parsing options: {}", match next_items.len() { @@ -760,8 +766,7 @@ pub fn parse( // then there is a syntax error. else if bb_items.is_empty() && next_items.is_empty() { return Failure( - parser.span, - parser.token.clone(), + parser.token.take(), "no rules expected this token in macro call", ); } @@ -781,7 +786,7 @@ pub fn parse( let match_cur = item.match_cur; item.push_match( match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, &ident.as_str()))), + MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), ); item.idx += 1; item.match_cur += 1; @@ -797,10 +802,9 @@ pub fn parse( /// The token is an identifier, but not `_`. /// We prohibit passing `_` to macros expecting `ident` for now. -fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> { - match *token { - token::Ident(ident, is_raw) if ident.name != keywords::Underscore.name() => - Some((ident, is_raw)), +fn get_macro_name(token: &Token) -> Option<(Name, bool)> { + match token.kind { + token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)), _ => None, } } @@ -809,7 +813,7 @@ fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> { /// /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that /// token. Be conservative (return true) if not sure. -fn may_begin_with(name: &str, token: &Token) -> bool { +fn may_begin_with(token: &Token, name: Name) -> bool { /// Checks whether the non-terminal may contain a single (non-keyword) identifier. fn may_be_ident(nt: &token::Nonterminal) -> bool { match *nt { @@ -819,18 +823,20 @@ fn may_begin_with(name: &str, token: &Token) -> bool { } match name { - "expr" => token.can_begin_expr(), - "ty" => token.can_begin_type(), - "ident" => get_macro_ident(token).is_some(), - "literal" => token.can_begin_literal_or_bool(), - "vis" => match *token { + sym::expr => token.can_begin_expr() + // This exception is here for backwards compatibility. + && !token.is_keyword(kw::Let), + sym::ty => token.can_begin_type(), + sym::ident => get_macro_name(token).is_some(), + sym::literal => token.can_begin_literal_or_bool(), + sym::vis => match token.kind { // The follow-set of :vis + "priv" keyword + interpolated - Token::Comma | Token::Ident(..) | Token::Interpolated(_) => true, + token::Comma | token::Ident(..) | token::Interpolated(_) => true, _ => token.can_begin_type(), }, - "block" => match *token { - Token::OpenDelim(token::Brace) => true, - Token::Interpolated(ref nt) => match **nt { + sym::block => match token.kind { + token::OpenDelim(token::Brace) => true, + token::Interpolated(ref nt) => match **nt { token::NtItem(_) | token::NtPat(_) | token::NtTy(_) @@ -842,39 +848,39 @@ fn may_begin_with(name: &str, token: &Token) -> bool { }, _ => false, }, - "path" | "meta" => match *token { - Token::ModSep | Token::Ident(..) => true, - Token::Interpolated(ref nt) => match **nt { + sym::path | sym::meta => match token.kind { + token::ModSep | token::Ident(..) => true, + token::Interpolated(ref nt) => match **nt { token::NtPath(_) | token::NtMeta(_) => true, _ => may_be_ident(&nt), }, _ => false, }, - "pat" => match *token { - Token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) - Token::OpenDelim(token::Paren) | // tuple pattern - Token::OpenDelim(token::Bracket) | // slice pattern - Token::BinOp(token::And) | // reference - Token::BinOp(token::Minus) | // negative literal - Token::AndAnd | // double reference - Token::Literal(..) | // literal - Token::DotDot | // range pattern (future compat) - Token::DotDotDot | // range pattern (future compat) - Token::ModSep | // path - Token::Lt | // path (UFCS constant) - Token::BinOp(token::Shl) => true, // path (double UFCS) - Token::Interpolated(ref nt) => may_be_ident(nt), + sym::pat => match token.kind { + token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) + token::OpenDelim(token::Paren) | // tuple pattern + token::OpenDelim(token::Bracket) | // slice pattern + token::BinOp(token::And) | // reference + token::BinOp(token::Minus) | // negative literal + token::AndAnd | // double reference + token::Literal(..) | // literal + token::DotDot | // range pattern (future compat) + token::DotDotDot | // range pattern (future compat) + token::ModSep | // path + token::Lt | // path (UFCS constant) + token::BinOp(token::Shl) => true, // path (double UFCS) + token::Interpolated(ref nt) => may_be_ident(nt), _ => false, }, - "lifetime" => match *token { - Token::Lifetime(_) => true, - Token::Interpolated(ref nt) => match **nt { + sym::lifetime => match token.kind { + token::Lifetime(_) => true, + token::Interpolated(ref nt) => match **nt { token::NtLifetime(_) | token::NtTT(_) => true, _ => false, }, _ => false, }, - _ => match *token { + _ => match token.kind { token::CloseDelim(_) => false, _ => true, }, @@ -893,46 +899,46 @@ fn may_begin_with(name: &str, token: &Token) -> bool { /// # Returns /// /// The parsed non-terminal. -fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal { - if name == "tt" { +fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Nonterminal { + if name == sym::tt { return token::NtTT(p.parse_token_tree()); } // check at the beginning and the parser checks after each bump p.process_potential_macro_variable(); match name { - "item" => match panictry!(p.parse_item()) { + sym::item => match panictry!(p.parse_item()) { Some(i) => token::NtItem(i), None => { p.fatal("expected an item keyword").emit(); FatalError.raise(); } }, - "block" => token::NtBlock(panictry!(p.parse_block())), - "stmt" => match panictry!(p.parse_stmt()) { + sym::block => token::NtBlock(panictry!(p.parse_block())), + sym::stmt => match panictry!(p.parse_stmt()) { Some(s) => token::NtStmt(s), None => { p.fatal("expected a statement").emit(); FatalError.raise(); } }, - "pat" => token::NtPat(panictry!(p.parse_pat(None))), - "expr" => token::NtExpr(panictry!(p.parse_expr())), - "literal" => token::NtLiteral(panictry!(p.parse_literal_maybe_minus())), - "ty" => token::NtTy(panictry!(p.parse_ty())), + sym::pat => token::NtPat(panictry!(p.parse_pat(None))), + sym::expr => token::NtExpr(panictry!(p.parse_expr())), + sym::literal => token::NtLiteral(panictry!(p.parse_literal_maybe_minus())), + sym::ty => token::NtTy(panictry!(p.parse_ty())), // this could be handled like a token, since it is one - "ident" => if let Some((ident, is_raw)) = get_macro_ident(&p.token) { - let span = p.span; + sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) { + let span = p.token.span; p.bump(); - token::NtIdent(Ident::new(ident.name, span), is_raw) + token::NtIdent(Ident::new(name, span), is_raw) } else { let token_str = pprust::token_to_string(&p.token); p.fatal(&format!("expected ident, found {}", &token_str)).emit(); FatalError.raise() } - "path" => token::NtPath(panictry!(p.parse_path_common(PathStyle::Type, false))), - "meta" => token::NtMeta(panictry!(p.parse_meta_item())), - "vis" => token::NtVis(panictry!(p.parse_visibility(true))), - "lifetime" => if p.check_lifetime() { + sym::path => token::NtPath(panictry!(p.parse_path(PathStyle::Type))), + sym::meta => token::NtMeta(panictry!(p.parse_meta_item())), + sym::vis => token::NtVis(panictry!(p.parse_visibility(true))), + sym::lifetime => if p.check_lifetime() { token::NtLifetime(p.expect_lifetime().ident) } else { let token_str = pprust::token_to_string(&p.token); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index bd64bb01021..7401f256412 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -1,35 +1,37 @@ -use crate::{ast, attr}; use crate::edition::Edition; -use crate::ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension}; -use crate::ext::base::{NormalTT, TTMacroExpander}; +use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind}; use crate::ext::expand::{AstFragment, AstFragmentKind}; -use crate::ext::tt::macro_parser::{Success, Error, Failure}; -use crate::ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal}; +use crate::ext::tt::macro_check; use crate::ext::tt::macro_parser::{parse, parse_failure_msg}; +use crate::ext::tt::macro_parser::{Error, Failure, Success}; +use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq}; use crate::ext::tt::quoted; use crate::ext::tt::transcribe::transcribe; use crate::feature_gate::Features; -use crate::parse::{Directory, ParseSess}; use crate::parse::parser::Parser; -use crate::parse::token::{self, NtTT}; -use crate::parse::token::Token::*; -use crate::symbol::Symbol; +use crate::parse::token::TokenKind::*; +use crate::parse::token::{self, NtTT, Token}; +use crate::parse::{Directory, ParseSess}; +use crate::symbol::{kw, sym, Symbol}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; +use crate::{ast, attr, attr::TransparencyError}; use errors::FatalError; -use syntax_pos::{Span, DUMMY_SP, symbol::Ident}; use log::debug; +use syntax_pos::Span; -use rustc_data_structures::fx::{FxHashMap}; +use rustc_data_structures::fx::FxHashMap; use std::borrow::Cow; use std::collections::hash_map::Entry; +use std::slice; -use rustc_data_structures::sync::Lrc; use errors::Applicability; +use rustc_data_structures::sync::Lrc; const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \ - `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, `literal`, \ - `path`, `meta`, `tt`, `item` and `vis`"; + `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \ + `literal`, `path`, `meta`, `tt`, `item` and `vis`"; pub struct ParserAnyMacro<'a> { parser: Parser<'a>, @@ -46,8 +48,9 @@ impl<'a> ParserAnyMacro<'a> { let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self; let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| { if parser.token == token::Eof && e.message().ends_with(", found `<eof>`") { - if !e.span.is_dummy() { // early end of macro arm (#52866) - e.replace_span_with(parser.sess.source_map().next_point(parser.span)); + if !e.span.is_dummy() { + // early end of macro arm (#52866) + e.replace_span_with(parser.sess.source_map().next_point(parser.token.span)); } let msg = &e.message[0]; e.message[0] = ( @@ -58,12 +61,13 @@ impl<'a> ParserAnyMacro<'a> { msg.1, ); } - if e.span.is_dummy() { // Get around lack of span in error (#30128) + if e.span.is_dummy() { + // Get around lack of span in error (#30128) e.replace_span_with(site_span); if parser.sess.source_map().span_to_filename(arm_span).is_real() { e.span_label(arm_span, "in this macro arm"); } - } else if !parser.sess.source_map().span_to_filename(parser.span).is_real() { + } else if !parser.sess.source_map().span_to_filename(parser.token.span).is_real() { e.span_label(site_span, "in this macro invocation"); } e @@ -85,6 +89,7 @@ impl<'a> ParserAnyMacro<'a> { struct MacroRulesMacroExpander { name: ast::Ident, + span: Span, lhses: Vec<quoted::TokenTree>, rhses: Vec<quoted::TokenTree>, valid: bool, @@ -96,18 +101,11 @@ impl TTMacroExpander for MacroRulesMacroExpander { cx: &'cx mut ExtCtxt<'_>, sp: Span, input: TokenStream, - def_span: Option<Span>, - ) -> Box<dyn MacResult+'cx> { + ) -> Box<dyn MacResult + 'cx> { if !self.valid { return DummyResult::any(sp); } - generic_extension(cx, - sp, - def_span, - self.name, - input, - &self.lhses, - &self.rhses) + generic_extension(cx, sp, self.span, self.name, input, &self.lhses, &self.rhses) } } @@ -117,27 +115,27 @@ fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { } /// Given `lhses` and `rhses`, this is the new macro we create -fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>, - sp: Span, - def_span: Option<Span>, - name: ast::Ident, - arg: TokenStream, - lhses: &[quoted::TokenTree], - rhses: &[quoted::TokenTree]) - -> Box<dyn MacResult+'cx> { +fn generic_extension<'cx>( + cx: &'cx mut ExtCtxt<'_>, + sp: Span, + def_span: Span, + name: ast::Ident, + arg: TokenStream, + lhses: &[quoted::TokenTree], + rhses: &[quoted::TokenTree], +) -> Box<dyn MacResult + 'cx> { if cx.trace_macros() { trace_macros_note(cx, sp, format!("expanding `{}! {{ {} }}`", name, arg)); } // Which arm's failure should we report? (the one furthest along) - let mut best_fail_spot = DUMMY_SP; - let mut best_fail_tok = None; - let mut best_fail_text = None; + let mut best_failure: Option<(Token, &str)> = None; - for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers + for (i, lhs) in lhses.iter().enumerate() { + // try each arm's matchers let lhs_tt = match *lhs { quoted::TokenTree::Delimited(_, ref delim) => &delim.tts[..], - _ => cx.span_bug(sp, "malformed macro lhs") + _ => cx.span_bug(sp, "malformed macro lhs"), }; match TokenTree::parse(cx, lhs_tt, arg.clone()) { @@ -151,7 +149,7 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>, let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>(); // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = transcribe(cx, Some(named_matches), rhs); + let mut tts = transcribe(cx, &named_matches, rhs); // Replace all the tokens for the corresponding positions in the macro, to maintain // proper positions in error reporting, while maintaining the macro_backtrace. @@ -172,9 +170,10 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>, path: Cow::from(cx.current_expansion.module.directory.as_path()), ownership: cx.current_expansion.directory_ownership, }; - let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false); - p.root_module_name = cx.current_expansion.module.mod_path.last() - .map(|id| id.as_str().to_string()); + let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); + p.root_module_name = + cx.current_expansion.module.mod_path.last().map(|id| id.as_str().to_string()); + p.last_type_ascription = cx.current_expansion.prior_type_ascription; p.process_potential_macro_variable(); // Let the context choose how to interpret the result. @@ -188,32 +187,28 @@ fn generic_extension<'cx>(cx: &'cx mut ExtCtxt<'_>, site_span: sp, macro_ident: name, arm_span, - }) + }); } - Failure(sp, tok, t) => if sp.lo() >= best_fail_spot.lo() { - best_fail_spot = sp; - best_fail_tok = Some(tok); - best_fail_text = Some(t); + Failure(token, msg) => match best_failure { + Some((ref best_token, _)) if best_token.span.lo() >= token.span.lo() => {} + _ => best_failure = Some((token, msg)), }, - Error(err_sp, ref msg) => { - cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]) - } + Error(err_sp, ref msg) => cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]), } } - let best_fail_msg = parse_failure_msg(best_fail_tok.expect("ran no matchers")); - let span = best_fail_spot.substitute_dummy(sp); - let mut err = cx.struct_span_err(span, &best_fail_msg); - err.span_label(span, best_fail_text.unwrap_or(&best_fail_msg)); - if let Some(sp) = def_span { - if cx.source_map().span_to_filename(sp).is_real() && !sp.is_dummy() { - err.span_label(cx.source_map().def_span(sp), "when calling this macro"); - } + let (token, label) = best_failure.expect("ran no matchers"); + let span = token.span.substitute_dummy(sp); + let mut err = cx.struct_span_err(span, &parse_failure_msg(&token)); + err.span_label(span, label); + if !def_span.is_dummy() && cx.source_map().span_to_filename(def_span).is_real() { + err.span_label(cx.source_map().def_span(def_span), "when calling this macro"); } // Check whether there's a missing comma in this macro call, like `println!("{}" a);` if let Some((arg, comma_span)) = arg.add_comma() { - for lhs in lhses { // try each arm's matchers + for lhs in lhses { + // try each arm's matchers let lhs_tt = match *lhs { quoted::TokenTree::Delimited(_, ref delim) => &delim.tts[..], _ => continue, @@ -250,10 +245,11 @@ pub fn compile( sess: &ParseSess, features: &Features, def: &ast::Item, - edition: Edition + edition: Edition, ) -> SyntaxExtension { - let lhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("lhs")); - let rhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("rhs")); + let lhs_nm = ast::Ident::new(sym::lhs, def.span); + let rhs_nm = ast::Ident::new(sym::rhs, def.span); + let tt_spec = ast::Ident::new(sym::tt, def.span); // Parse the macro_rules! invocation let body = match def.node { @@ -267,32 +263,44 @@ pub fn compile( // ...quasiquoting this would be nice. // These spans won't matter, anyways let argument_gram = vec![ - quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition { - tts: vec![ - quoted::TokenTree::MetaVarDecl(DUMMY_SP, lhs_nm, ast::Ident::from_str("tt")), - quoted::TokenTree::Token(DUMMY_SP, token::FatArrow), - quoted::TokenTree::MetaVarDecl(DUMMY_SP, rhs_nm, ast::Ident::from_str("tt")), - ], - separator: Some(if body.legacy { token::Semi } else { token::Comma }), - op: quoted::KleeneOp::OneOrMore, - num_captures: 2, - })), + quoted::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(quoted::SequenceRepetition { + tts: vec![ + quoted::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec), + quoted::TokenTree::token(token::FatArrow, def.span), + quoted::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec), + ], + separator: Some(Token::new( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )), + kleene: quoted::KleeneToken::new(quoted::KleeneOp::OneOrMore, def.span), + num_captures: 2, + }), + ), // to phase into semicolon-termination instead of semicolon-separation - quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition { - tts: vec![quoted::TokenTree::Token(DUMMY_SP, token::Semi)], - separator: None, - op: quoted::KleeneOp::ZeroOrMore, - num_captures: 0 - })), + quoted::TokenTree::Sequence( + DelimSpan::dummy(), + Lrc::new(quoted::SequenceRepetition { + tts: vec![quoted::TokenTree::token( + if body.legacy { token::Semi } else { token::Comma }, + def.span, + )], + separator: None, + kleene: quoted::KleeneToken::new(quoted::KleeneOp::ZeroOrMore, def.span), + num_captures: 0, + }), + ), ]; let argument_map = match parse(sess, body.stream(), &argument_gram, None, true) { Success(m) => m, - Failure(sp, tok, t) => { - let s = parse_failure_msg(tok); - let sp = sp.substitute_dummy(def.span); + Failure(token, msg) => { + let s = parse_failure_msg(&token); + let sp = token.span.substitute_dummy(def.span); let mut err = sess.span_diagnostic.struct_span_fatal(sp, &s); - err.span_label(sp, t); + err.span_label(sp, msg); err.emit(); FatalError.raise(); } @@ -304,9 +312,10 @@ pub fn compile( let mut valid = true; // Extract the arguments: - let lhses = match *argument_map[&lhs_nm] { - MatchedSeq(ref s, _) => { - s.iter().map(|m| { + let lhses = match argument_map[&lhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { if let MatchedNonterminal(ref nt) = *m { if let NtTT(ref tt) = **nt { let tt = quoted::parse( @@ -325,14 +334,15 @@ pub fn compile( } } sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }).collect::<Vec<quoted::TokenTree>>() - } - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") + }) + .collect::<Vec<quoted::TokenTree>>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"), }; - let rhses = match *argument_map[&rhs_nm] { - MatchedSeq(ref s, _) => { - s.iter().map(|m| { + let rhses = match argument_map[&rhs_nm] { + MatchedSeq(ref s, _) => s + .iter() + .map(|m| { if let MatchedNonterminal(ref nt) = *m { if let NtTT(ref tt) = **nt { return quoted::parse( @@ -343,14 +353,15 @@ pub fn compile( &def.attrs, edition, def.id, - ).pop() - .unwrap(); + ) + .pop() + .unwrap(); } } sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs") - }).collect::<Vec<quoted::TokenTree>>() - } - _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs") + }) + .collect::<Vec<quoted::TokenTree>>(), + _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"), }; for rhs in &rhses { @@ -359,83 +370,90 @@ pub fn compile( // don't abort iteration early, so that errors for multiple lhses can be reported for lhs in &lhses { - valid &= check_lhs_no_empty_seq(sess, &[lhs.clone()]); - valid &= check_lhs_duplicate_matcher_bindings( - sess, - &[lhs.clone()], - &mut FxHashMap::default(), - def.id - ); + valid &= check_lhs_no_empty_seq(sess, slice::from_ref(lhs)); } - let expander: Box<_> = Box::new(MacroRulesMacroExpander { - name: def.ident, - lhses, - rhses, - valid, - }); - - if body.legacy { - let allow_internal_unstable = attr::find_by_name(&def.attrs, "allow_internal_unstable") - .map(|attr| attr - .meta_item_list() - .map(|list| list.iter() - .map(|it| it.name().unwrap_or_else(|| sess.span_diagnostic.span_bug( - it.span, "allow internal unstable expects feature names", - ))) - .collect::<Vec<Symbol>>().into() - ) + // We use CRATE_NODE_ID instead of `def.id` otherwise we may emit buffered lints for a node id + // that is not lint-checked and trigger the "failed to process buffered lint here" bug. + valid &= macro_check::check_meta_variables(sess, ast::CRATE_NODE_ID, def.span, &lhses, &rhses); + + let expander: Box<_> = + Box::new(MacroRulesMacroExpander { name: def.ident, span: def.span, lhses, rhses, valid }); + + let (default_transparency, transparency_error) = + attr::find_transparency(&def.attrs, body.legacy); + match transparency_error { + Some(TransparencyError::UnknownTransparency(value, span)) => + sess.span_diagnostic.span_err( + span, &format!("unknown macro transparency: `{}`", value) + ), + Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => + sess.span_diagnostic.span_err( + vec![old_span, new_span], "multiple macro transparency attributes" + ), + None => {} + } + + let allow_internal_unstable = + attr::find_by_name(&def.attrs, sym::allow_internal_unstable).map(|attr| { + attr.meta_item_list() + .map(|list| { + list.iter() + .filter_map(|it| { + let name = it.ident().map(|ident| ident.name); + if name.is_none() { + sess.span_diagnostic.span_err( + it.span(), + "allow internal unstable expects feature names", + ) + } + name + }) + .collect::<Vec<Symbol>>() + .into() + }) .unwrap_or_else(|| { sess.span_diagnostic.span_warn( - attr.span, "allow_internal_unstable expects list of feature names. In the \ - future this will become a hard error. Please use `allow_internal_unstable(\ - foo, bar)` to only allow the `foo` and `bar` features", + attr.span, + "allow_internal_unstable expects list of feature names. In the \ + future this will become a hard error. Please use `allow_internal_unstable(\ + foo, bar)` to only allow the `foo` and `bar` features", ); - vec![Symbol::intern("allow_internal_unstable_backcompat_hack")].into() + vec![sym::allow_internal_unstable_backcompat_hack].into() }) - ); - let allow_internal_unsafe = attr::contains_name(&def.attrs, "allow_internal_unsafe"); - let mut local_inner_macros = false; - if let Some(macro_export) = attr::find_by_name(&def.attrs, "macro_export") { - if let Some(l) = macro_export.meta_item_list() { - local_inner_macros = attr::list_contains_name(&l, "local_inner_macros"); - } - } - - let unstable_feature = attr::find_stability(&sess, - &def.attrs, def.span).and_then(|stability| { - if let attr::StabilityLevel::Unstable { issue, .. } = stability.level { - Some((stability.feature, issue)) - } else { - None - } }); - NormalTT { - expander, - def_info: Some((def.id, def.span)), - allow_internal_unstable, - allow_internal_unsafe, - local_inner_macros, - unstable_feature, - edition, + let mut local_inner_macros = false; + if let Some(macro_export) = attr::find_by_name(&def.attrs, sym::macro_export) { + if let Some(l) = macro_export.meta_item_list() { + local_inner_macros = attr::list_contains_name(&l, sym::local_inner_macros); } - } else { - let is_transparent = attr::contains_name(&def.attrs, "rustc_transparent_macro"); + } - SyntaxExtension::DeclMacro { - expander, - def_info: Some((def.id, def.span)), - is_transparent, - edition, - } + let is_builtin = attr::contains_name(&def.attrs, sym::rustc_builtin_macro); + + SyntaxExtension { + kind: SyntaxExtensionKind::LegacyBang(expander), + span: def.span, + default_transparency, + allow_internal_unstable, + allow_internal_unsafe: attr::contains_name(&def.attrs, sym::allow_internal_unsafe), + local_inner_macros, + stability: attr::find_stability(&sess, &def.attrs, def.span), + deprecation: attr::find_deprecation(&sess, &def.attrs, def.span), + helper_attrs: Vec::new(), + edition, + is_builtin, + is_derive_copy: is_builtin && def.ident.name == sym::Copy, } } -fn check_lhs_nt_follows(sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - lhs: "ed::TokenTree) -> bool { +fn check_lhs_nt_follows( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + lhs: "ed::TokenTree, +) -> bool { // lhs is going to be like TokenTree::Delimited(...), where the // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens. if let quoted::TokenTree::Delimited(_, ref tts) = *lhs { @@ -456,19 +474,22 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool { for tt in tts { match *tt { TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (), - TokenTree::Delimited(_, ref del) => if !check_lhs_no_empty_seq(sess, &del.tts) { - return false; - }, + TokenTree::Delimited(_, ref del) => { + if !check_lhs_no_empty_seq(sess, &del.tts) { + return false; + } + } TokenTree::Sequence(span, ref seq) => { - if seq.separator.is_none() && seq.tts.iter().all(|seq_tt| { - match *seq_tt { - TokenTree::MetaVarDecl(_, _, id) => id.name == "vis", - TokenTree::Sequence(_, ref sub_seq) => - sub_seq.op == quoted::KleeneOp::ZeroOrMore - || sub_seq.op == quoted::KleeneOp::ZeroOrOne, + if seq.separator.is_none() + && seq.tts.iter().all(|seq_tt| match *seq_tt { + TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis, + TokenTree::Sequence(_, ref sub_seq) => { + sub_seq.kleene.op == quoted::KleeneOp::ZeroOrMore + || sub_seq.kleene.op == quoted::KleeneOp::ZeroOrOne + } _ => false, - } - }) { + }) + { let sp = span.entire(); sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); return false; @@ -483,65 +504,20 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool { true } -/// Check that the LHS contains no duplicate matcher bindings. e.g. `$a:expr, $a:expr` would be -/// illegal, since it would be ambiguous which `$a` to use if we ever needed to. -fn check_lhs_duplicate_matcher_bindings( - sess: &ParseSess, - tts: &[quoted::TokenTree], - metavar_names: &mut FxHashMap<Ident, Span>, - node_id: ast::NodeId, -) -> bool { - use self::quoted::TokenTree; - use crate::early_buffered_lints::BufferedEarlyLintId; - for tt in tts { - match *tt { - TokenTree::MetaVarDecl(span, name, _kind) => { - if let Some(&prev_span) = metavar_names.get(&name) { - // FIXME(mark-i-m): in a few cycles, make this a hard error. - // sess.span_diagnostic - // .struct_span_err(span, "duplicate matcher binding") - // .span_note(prev_span, "previous declaration was here") - // .emit(); - sess.buffer_lint( - BufferedEarlyLintId::DuplicateMacroMatcherBindingName, - crate::source_map::MultiSpan::from(vec![prev_span, span]), - node_id, - "duplicate matcher binding" - ); - return false; - } else { - metavar_names.insert(name, span); - } - } - TokenTree::Delimited(_, ref del) => { - if !check_lhs_duplicate_matcher_bindings(sess, &del.tts, metavar_names, node_id) { - return false; - } - }, - TokenTree::Sequence(_, ref seq) => { - if !check_lhs_duplicate_matcher_bindings(sess, &seq.tts, metavar_names, node_id) { - return false; - } - } - _ => {} - } - } - - true -} - fn check_rhs(sess: &ParseSess, rhs: "ed::TokenTree) -> bool { match *rhs { quoted::TokenTree::Delimited(..) => return true, - _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited") + _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"), } false } -fn check_matcher(sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - matcher: &[quoted::TokenTree]) -> bool { +fn check_matcher( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + matcher: &[quoted::TokenTree], +) -> bool { let first_sets = FirstSets::new(matcher); let empty_suffix = TokenSet::empty(); let err = sess.span_diagnostic.err_count(); @@ -614,15 +590,14 @@ impl FirstSets { // If the sequence contents can be empty, then the first // token could be the separator token itself. - if let (Some(ref sep), true) = (seq_rep.separator.clone(), - subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone())); + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); } // Reverse scan: Sequence comes before `first`. if subfirst.maybe_empty - || seq_rep.op == quoted::KleeneOp::ZeroOrMore - || seq_rep.op == quoted::KleeneOp::ZeroOrOne + || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrOne { // If sequence is potentially empty, then // union them (preserving first emptiness). @@ -658,40 +633,37 @@ impl FirstSets { return first; } TokenTree::Sequence(sp, ref seq_rep) => { - match self.first.get(&sp.entire()) { - Some(&Some(ref subfirst)) => { - - // If the sequence contents can be empty, then the first - // token could be the separator token itself. - - if let (Some(ref sep), true) = (seq_rep.separator.clone(), - subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone())); - } - - assert!(first.maybe_empty); - first.add_all(subfirst); - if subfirst.maybe_empty - || seq_rep.op == quoted::KleeneOp::ZeroOrMore - || seq_rep.op == quoted::KleeneOp::ZeroOrOne - { - // continue scanning for more first - // tokens, but also make sure we - // restore empty-tracking state - first.maybe_empty = true; - continue; - } else { - return first; - } - } - + let subfirst_owned; + let subfirst = match self.first.get(&sp.entire()) { + Some(&Some(ref subfirst)) => subfirst, Some(&None) => { - panic!("assume all sequences have (unique) spans for now"); + subfirst_owned = self.first(&seq_rep.tts[..]); + &subfirst_owned } - None => { panic!("We missed a sequence during FirstSets construction"); } + }; + + // If the sequence contents can be empty, then the first + // token could be the separator token itself. + if let (Some(sep), true) = (&seq_rep.separator, subfirst.maybe_empty) { + first.add_one_maybe(TokenTree::Token(sep.clone())); + } + + assert!(first.maybe_empty); + first.add_all(subfirst); + if subfirst.maybe_empty + || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrMore + || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrOne + { + // Continue scanning for more first + // tokens, but also make sure we + // restore empty-tracking state. + first.maybe_empty = true; + continue; + } else { + return first; } } } @@ -722,7 +694,9 @@ struct TokenSet { impl TokenSet { // Returns a set for the empty sequence. - fn empty() -> Self { TokenSet { tokens: Vec::new(), maybe_empty: true } } + fn empty() -> Self { + TokenSet { tokens: Vec::new(), maybe_empty: true } + } // Returns the set `{ tok }` for the single-token (and thus // non-empty) sequence [tok]. @@ -781,7 +755,7 @@ impl TokenSet { } // Checks that `matcher` is internally consistent and that it -// can legally by followed by a token N, for all N in `follow`. +// can legally be followed by a token `N`, for all `N` in `follow`. // (If `follow` is empty, then it imposes no constraint on // the `matcher`.) // @@ -791,12 +765,14 @@ impl TokenSet { // // Requires that `first_sets` is pre-computed for `matcher`; // see `FirstSets::new`. -fn check_matcher_core(sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - first_sets: &FirstSets, - matcher: &[quoted::TokenTree], - follow: &TokenSet) -> TokenSet { +fn check_matcher_core( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + first_sets: &FirstSets, + matcher: &[quoted::TokenTree], + follow: &TokenSet, +) -> TokenSet { use quoted::TokenTree; let mut last = TokenSet::empty(); @@ -806,11 +782,13 @@ fn check_matcher_core(sess: &ParseSess, // then ensure T can also be followed by any element of FOLLOW. 'each_token: for i in 0..matcher.len() { let token = &matcher[i]; - let suffix = &matcher[i+1..]; + let suffix = &matcher[i + 1..]; let build_suffix_first = || { let mut s = first_sets.first(suffix); - if s.maybe_empty { s.add_all(follow); } + if s.maybe_empty { + s.add_all(follow); + } s }; @@ -826,7 +804,8 @@ fn check_matcher_core(sess: &ParseSess, let can_be_followed_by_any; if let Err(bad_frag) = has_legal_fragment_specifier(sess, features, attrs, token) { let msg = format!("invalid fragment specifier `{}`", bad_frag); - sess.span_diagnostic.struct_span_err(token.span(), &msg) + sess.span_diagnostic + .struct_span_err(token.span(), &msg) .help(VALID_FRAGMENT_NAMES_MSG) .emit(); // (This eliminates false positives and duplicates @@ -857,7 +836,7 @@ fn check_matcher_core(sess: &ParseSess, // against SUFFIX continue 'each_token; } - TokenTree::Sequence(sp, ref seq_rep) => { + TokenTree::Sequence(_, ref seq_rep) => { suffix_first = build_suffix_first(); // The trick here: when we check the interior, we want // to include the separator (if any) as a potential @@ -870,9 +849,9 @@ fn check_matcher_core(sess: &ParseSess, // work of cloning it? But then again, this way I may // get a "tighter" span? let mut new; - let my_suffix = if let Some(ref u) = seq_rep.separator { + let my_suffix = if let Some(sep) = &seq_rep.separator { new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sp.entire(), u.clone())); + new.add_one_maybe(TokenTree::Token(sep.clone())); &new } else { &suffix_first @@ -881,12 +860,8 @@ fn check_matcher_core(sess: &ParseSess, // At this point, `suffix_first` is built, and // `my_suffix` is some TokenSet that we can use // for checking the interior of `seq_rep`. - let next = check_matcher_core(sess, - features, - attrs, - first_sets, - &seq_rep.tts, - my_suffix); + let next = + check_matcher_core(sess, features, attrs, first_sets, &seq_rep.tts, my_suffix); if next.maybe_empty { last.add_all(&next); } else { @@ -908,16 +883,17 @@ fn check_matcher_core(sess: &ParseSess, for next_token in &suffix_first.tokens { match is_in_follow(next_token, &frag_spec.as_str()) { IsInFollow::Invalid(msg, help) => { - sess.span_diagnostic.struct_span_err(next_token.span(), &msg) - .help(help).emit(); + sess.span_diagnostic + .struct_span_err(next_token.span(), &msg) + .help(help) + .emit(); // don't bother reporting every source of // conflict for a particular element of `last`. continue 'each_last; } IsInFollow::Yes => {} - IsInFollow::No(ref possible) => { - let may_be = if last.tokens.len() == 1 && - suffix_first.tokens.len() == 1 + IsInFollow::No(possible) => { + let may_be = if last.tokens.len() == 1 && suffix_first.tokens.len() == 1 { "is" } else { @@ -927,33 +903,37 @@ fn check_matcher_core(sess: &ParseSess, let sp = next_token.span(); let mut err = sess.span_diagnostic.struct_span_err( sp, - &format!("`${name}:{frag}` {may_be} followed by `{next}`, which \ - is not allowed for `{frag}` fragments", - name=name, - frag=frag_spec, - next=quoted_tt_to_string(next_token), - may_be=may_be), + &format!( + "`${name}:{frag}` {may_be} followed by `{next}`, which \ + is not allowed for `{frag}` fragments", + name = name, + frag = frag_spec, + next = quoted_tt_to_string(next_token), + may_be = may_be + ), ); err.span_label( sp, format!("not allowed after `{}` fragments", frag_spec), ); let msg = "allowed there are: "; - match &possible[..] { + match possible { &[] => {} &[t] => { err.note(&format!( "only {} is allowed after `{}` fragments", - t, - frag_spec, + t, frag_spec, )); } ts => { err.note(&format!( "{}{} or {}", msg, - ts[..ts.len() - 1].iter().map(|s| *s) - .collect::<Vec<_>>().join(", "), + ts[..ts.len() - 1] + .iter() + .map(|s| *s) + .collect::<Vec<_>>() + .join(", "), ts[ts.len() - 1], )); } @@ -1003,7 +983,7 @@ fn frag_can_be_followed_by_any(frag: &str) -> bool { enum IsInFollow { Yes, - No(Vec<&'static str>), + No(&'static [&'static str]), Invalid(String, &'static str), } @@ -1018,7 +998,7 @@ enum IsInFollow { fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> IsInFollow { use quoted::TokenTree; - if let TokenTree::Token(_, token::CloseDelim(_)) = *tok { + if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok { // closing a token tree can never be matched by any fragment; // iow, we always require that `(` and `)` match, etc. IsInFollow::Yes @@ -1028,110 +1008,131 @@ fn is_in_follow(tok: "ed::TokenTree, frag: &str) -> IsInFollow { // since items *must* be followed by either a `;` or a `}`, we can // accept anything after them IsInFollow::Yes - }, + } "block" => { // anything can follow block, the braces provide an easy boundary to // maintain IsInFollow::Yes - }, - "stmt" | "expr" => { - let tokens = vec!["`=>`", "`,`", "`;`"]; - match *tok { - TokenTree::Token(_, ref tok) => match *tok { + } + "stmt" | "expr" => { + const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; + match tok { + TokenTree::Token(token) => match token.kind { FatArrow | Comma | Semi => IsInFollow::Yes, - _ => IsInFollow::No(tokens), + _ => IsInFollow::No(TOKENS), }, - _ => IsInFollow::No(tokens), + _ => IsInFollow::No(TOKENS), } - }, + } "pat" => { - let tokens = vec!["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; - match *tok { - TokenTree::Token(_, ref tok) => match *tok { + const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; + match tok { + TokenTree::Token(token) => match token.kind { FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes, - Ident(i, false) if i.name == "if" || i.name == "in" => IsInFollow::Yes, - _ => IsInFollow::No(tokens), + Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes, + _ => IsInFollow::No(TOKENS), }, - _ => IsInFollow::No(tokens), + _ => IsInFollow::No(TOKENS), } - }, + } "path" | "ty" => { - let tokens = vec![ - "`{`", "`[`", "`=>`", "`,`", "`>`","`=`", "`:`", "`;`", "`|`", "`as`", + const TOKENS: &[&str] = &[ + "`{`", "`[`", "`=>`", "`,`", "`>`", "`=`", "`:`", "`;`", "`|`", "`as`", "`where`", ]; - match *tok { - TokenTree::Token(_, ref tok) => match *tok { - OpenDelim(token::DelimToken::Brace) | - OpenDelim(token::DelimToken::Bracket) | - Comma | FatArrow | Colon | Eq | Gt | BinOp(token::Shr) | Semi | - BinOp(token::Or) => IsInFollow::Yes, - Ident(i, false) if i.name == "as" || i.name == "where" => IsInFollow::Yes, - _ => IsInFollow::No(tokens), + match tok { + TokenTree::Token(token) => match token.kind { + OpenDelim(token::DelimToken::Brace) + | OpenDelim(token::DelimToken::Bracket) + | Comma + | FatArrow + | Colon + | Eq + | Gt + | BinOp(token::Shr) + | Semi + | BinOp(token::Or) => IsInFollow::Yes, + Ident(name, false) if name == kw::As || name == kw::Where => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == "block" => IsInFollow::Yes, - _ => IsInFollow::No(tokens), + TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block => { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), } - }, + } "ident" | "lifetime" => { // being a single token, idents and lifetimes are harmless IsInFollow::Yes - }, + } "literal" => { // literals may be of a single token, or two tokens (negative numbers) IsInFollow::Yes - }, + } "meta" | "tt" => { // being either a single token or a delimited sequence, tt is // harmless IsInFollow::Yes - }, + } "vis" => { // Explicitly disallow `priv`, on the off chance it comes back. - let tokens = vec!["`,`", "an ident", "a type"]; - match *tok { - TokenTree::Token(_, ref tok) => match *tok { + const TOKENS: &[&str] = &["`,`", "an ident", "a type"]; + match tok { + TokenTree::Token(token) => match token.kind { Comma => IsInFollow::Yes, - Ident(i, is_raw) if is_raw || i.name != "priv" => IsInFollow::Yes, - ref tok => if tok.can_begin_type() { - IsInFollow::Yes - } else { - IsInFollow::No(tokens) + Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes, + _ => { + if token.can_begin_type() { + IsInFollow::Yes + } else { + IsInFollow::No(TOKENS) + } } }, - TokenTree::MetaVarDecl(_, _, frag) if frag.name == "ident" - || frag.name == "ty" - || frag.name == "path" => IsInFollow::Yes, - _ => IsInFollow::No(tokens), + TokenTree::MetaVarDecl(_, _, frag) + if frag.name == sym::ident + || frag.name == sym::ty + || frag.name == sym::path => + { + IsInFollow::Yes + } + _ => IsInFollow::No(TOKENS), } - }, - "" => IsInFollow::Yes, // keywords::Invalid - _ => IsInFollow::Invalid(format!("invalid fragment specifier `{}`", frag), - VALID_FRAGMENT_NAMES_MSG), + } + "" => IsInFollow::Yes, // kw::Invalid + _ => IsInFollow::Invalid( + format!("invalid fragment specifier `{}`", frag), + VALID_FRAGMENT_NAMES_MSG, + ), } } } -fn has_legal_fragment_specifier(sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - tok: "ed::TokenTree) -> Result<(), String> { +fn has_legal_fragment_specifier( + sess: &ParseSess, + features: &Features, + attrs: &[ast::Attribute], + tok: "ed::TokenTree, +) -> Result<(), String> { debug!("has_legal_fragment_specifier({:?})", tok); if let quoted::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok { - let frag_name = frag_spec.as_str(); let frag_span = tok.span(); - if !is_legal_fragment_specifier(sess, features, attrs, &frag_name, frag_span) { - return Err(frag_name.to_string()); + if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) { + return Err(frag_spec.to_string()); } } Ok(()) } -fn is_legal_fragment_specifier(_sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - frag_name: &str, - _frag_span: Span) -> bool { +fn is_legal_fragment_specifier( + _sess: &ParseSess, + _features: &Features, + _attrs: &[ast::Attribute], + frag_name: Symbol, + _frag_span: Span, +) -> bool { /* * If new fragment specifiers are invented in nightly, `_sess`, * `_features`, `_attrs`, and `_frag_span` will be useful here @@ -1139,19 +1140,32 @@ fn is_legal_fragment_specifier(_sess: &ParseSess, * this function. */ match frag_name { - "item" | "block" | "stmt" | "expr" | "pat" | "lifetime" | - "path" | "ty" | "ident" | "meta" | "tt" | "vis" | "literal" | - "" => true, + sym::item + | sym::block + | sym::stmt + | sym::expr + | sym::pat + | sym::lifetime + | sym::path + | sym::ty + | sym::ident + | sym::meta + | sym::tt + | sym::vis + | sym::literal + | kw::Invalid => true, _ => false, } } fn quoted_tt_to_string(tt: "ed::TokenTree) -> String { match *tt { - quoted::TokenTree::Token(_, ref tok) => crate::print::pprust::token_to_string(tok), + quoted::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token), quoted::TokenTree::MetaVar(_, name) => format!("${}", name), quoted::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind), - _ => panic!("unexpected quoted::TokenTree::{{Sequence or Delimited}} \ - in follow set checker"), + _ => panic!( + "unexpected quoted::TokenTree::{{Sequence or Delimited}} \ + in follow set checker" + ), } } diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index b24edb57e52..cad94a0e4c1 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -1,12 +1,12 @@ +use crate::ast; use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; use crate::ext::tt::macro_parser; use crate::feature_gate::Features; -use crate::parse::{token, ParseSess}; +use crate::parse::token::{self, Token, TokenKind}; +use crate::parse::ParseSess; use crate::print::pprust; +use crate::symbol::kw; use crate::tokenstream::{self, DelimSpan}; -use crate::ast; -use crate::symbol::keywords; use syntax_pos::{edition::Edition, BytePos, Span}; @@ -22,24 +22,14 @@ pub struct Delimited { } impl Delimited { - /// Returns the opening delimiter (possibly `NoDelim`). - pub fn open_token(&self) -> token::Token { - token::OpenDelim(self.delim) - } - - /// Returns the closing delimiter (possibly `NoDelim`). - pub fn close_token(&self) -> token::Token { - token::CloseDelim(self.delim) - } - /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter. pub fn open_tt(&self, span: Span) -> TokenTree { let open_span = if span.is_dummy() { span } else { - span.with_lo(span.lo() + BytePos(self.delim.len() as u32)) + span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) }; - TokenTree::Token(open_span, self.open_token()) + TokenTree::token(token::OpenDelim(self.delim), open_span) } /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter. @@ -49,7 +39,7 @@ impl Delimited { } else { span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) }; - TokenTree::Token(close_span, self.close_token()) + TokenTree::token(token::CloseDelim(self.delim), close_span) } } @@ -58,13 +48,25 @@ pub struct SequenceRepetition { /// The sequence of token trees pub tts: Vec<TokenTree>, /// The optional separator - pub separator: Option<token::Token>, + pub separator: Option<Token>, /// Whether the sequence can be repeated zero (*), or one or more times (+) - pub op: KleeneOp, + pub kleene: KleeneToken, /// The number of `Match`s that appear in the sequence (and subsequences) pub num_captures: usize, } +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] +pub struct KleeneToken { + pub span: Span, + pub op: KleeneOp, +} + +impl KleeneToken { + pub fn new(op: KleeneOp, span: Span) -> KleeneToken { + KleeneToken { span, op } + } +} + /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) /// for token sequences. #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] @@ -73,6 +75,7 @@ pub enum KleeneOp { ZeroOrMore, /// Kleene plus (`+`) for one or more repetitions OneOrMore, + /// Kleene optional (`?`) for zero or one reptitions ZeroOrOne, } @@ -80,7 +83,7 @@ pub enum KleeneOp { /// are "first-class" token trees. Useful for parsing macros. #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum TokenTree { - Token(Span, token::Token), + Token(Token), Delimited(DelimSpan, Lrc<Delimited>), /// A kleene-style repetition sequence Sequence(DelimSpan, Lrc<SequenceRepetition>), @@ -120,6 +123,22 @@ impl TokenTree { } } + /// Returns `true` if the given token tree is delimited. + pub fn is_delimited(&self) -> bool { + match *self { + TokenTree::Delimited(..) => true, + _ => false, + } + } + + /// Returns `true` if the given token tree is a token of the given kind. + pub fn is_token(&self, expected_kind: &TokenKind) -> bool { + match self { + TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind, + _ => false, + } + } + /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences. pub fn get_tt(&self, index: usize) -> TokenTree { match (self, index) { @@ -143,13 +162,16 @@ impl TokenTree { /// Retrieves the `TokenTree`'s span. pub fn span(&self) -> Span { match *self { - TokenTree::Token(sp, _) - | TokenTree::MetaVar(sp, _) - | TokenTree::MetaVarDecl(sp, _, _) => sp, - TokenTree::Delimited(sp, _) - | TokenTree::Sequence(sp, _) => sp.entire(), + TokenTree::Token(Token { span, .. }) + | TokenTree::MetaVar(span, _) + | TokenTree::MetaVarDecl(span, _, _) => span, + TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(), } } + + crate fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } } /// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this @@ -204,31 +226,23 @@ pub fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() { - Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() { - Some((kind, _)) => { - let span = end_sp.with_lo(start_sp.lo()); - result.push(TokenTree::MetaVarDecl(span, ident, kind)); - continue; - } - _ => end_sp, - }, - tree => tree - .as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(span), - }, - tree => tree - .as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(start_sp), + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + match trees.next() { + Some(tokenstream::TokenTree::Token(token)) => match token.ident() { + Some((kind, _)) => { + let span = token.span.with_lo(start_sp.lo()); + result.push(TokenTree::MetaVarDecl(span, ident, kind)); + continue; + } + _ => token.span, + }, + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), + } + } + tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), }; sess.missing_fragment_specifiers.borrow_mut().insert(span); - result.push(TokenTree::MetaVarDecl( - span, - ident, - keywords::Invalid.ident(), - )); + result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); } // Not a metavar or no matchers allowed, so just return the tree @@ -253,29 +267,26 @@ pub fn parse( /// - `sess`: the parsing session. Any errors will be emitted to this session. /// - `features`, `attrs`: language feature flags and attributes so that we know whether to use /// unstable features or not. -fn parse_tree<I>( +fn parse_tree( tree: tokenstream::TokenTree, - trees: &mut Peekable<I>, + trees: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>, expect_matchers: bool, sess: &ParseSess, features: &Features, attrs: &[ast::Attribute], edition: Edition, macro_node_id: NodeId, -) -> TokenTree -where - I: Iterator<Item = tokenstream::TokenTree>, -{ +) -> TokenTree { // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() { + tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { // `tree` is followed by a delimited set of token trees. This indicates the beginning // of a repetition sequence in the macro (e.g. `$(pat)*`). Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { // Must have `(` not `{` or `[` if delim != token::Paren { - let tok = pprust::token_to_string(&token::OpenDelim(delim)); + let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); let msg = format!("expected `(`, found `{}`", tok); sess.span_diagnostic.span_err(span.entire(), &msg); } @@ -290,16 +301,7 @@ where macro_node_id, ); // Get the Kleene operator and optional separator - let (separator, op) = - parse_sep_and_kleene_op( - trees, - span.entire(), - sess, - features, - attrs, - edition, - macro_node_id, - ); + let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); // Count the number of captured "names" (i.e., named metavars) let name_captures = macro_parser::count_names(&sequence); TokenTree::Sequence( @@ -307,7 +309,7 @@ where Lrc::new(SequenceRepetition { tts: sequence, separator, - op, + kleene, num_captures: name_captures, }), ) @@ -315,40 +317,37 @@ where // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special // metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => { + Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { let (ident, is_raw) = token.ident().unwrap(); - let span = ident_span.with_lo(span.lo()); - if ident.name == keywords::Crate.name() && !is_raw { - let ident = ast::Ident::new(keywords::DollarCrate.name(), ident.span); - TokenTree::Token(span, token::Ident(ident, is_raw)) + let span = ident.span.with_lo(span.lo()); + if ident.name == kw::Crate && !is_raw { + TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) } else { TokenTree::MetaVar(span, ident) } } // `tree` is followed by a random token. This is an error. - Some(tokenstream::TokenTree::Token(span, tok)) => { - let msg = format!( - "expected identifier, found `{}`", - pprust::token_to_string(&tok) - ); - sess.span_diagnostic.span_err(span, &msg); - TokenTree::MetaVar(span, keywords::Invalid.ident()) + Some(tokenstream::TokenTree::Token(token)) => { + let msg = + format!("expected identifier, found `{}`", pprust::token_to_string(&token),); + sess.span_diagnostic.span_err(token.span, &msg); + TokenTree::MetaVar(token.span, ast::Ident::invalid()) } // There are no more tokens. Just return the `$` we already have. - None => TokenTree::Token(span, token::Dollar), + None => TokenTree::token(token::Dollar, span), }, // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok), + tokenstream::TokenTree::Token(token) => TokenTree::Token(token), // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // descend into the delimited set and further parse it. tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( span, Lrc::new(Delimited { - delim: delim, + delim, tts: parse( tts.into(), expect_matchers, @@ -365,8 +364,8 @@ where /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return /// `None`. -fn kleene_op(token: &token::Token) -> Option<KleeneOp> { - match *token { +fn kleene_op(token: &Token) -> Option<KleeneOp> { + match token.kind { token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), token::Question => Some(KleeneOp::ZeroOrOne), @@ -379,22 +378,16 @@ fn kleene_op(token: &token::Token) -> Option<KleeneOp> { /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp /// - Err(span) if the next token tree is not a token -fn parse_kleene_op<I>( - input: &mut I, +fn parse_kleene_op( + input: &mut impl Iterator<Item = tokenstream::TokenTree>, span: Span, -) -> Result<Result<(KleeneOp, Span), (token::Token, Span)>, Span> -where - I: Iterator<Item = tokenstream::TokenTree>, -{ +) -> Result<Result<(KleeneOp, Span), Token>, Span> { match input.next() { - Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) { - Some(op) => Ok(Ok((op, span))), - None => Ok(Err((tok, span))), + Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { + Some(op) => Ok(Ok((op, token.span))), + None => Ok(Err(token)), }, - tree => Err(tree - .as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(span)), + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), } } @@ -410,196 +403,35 @@ where /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// error with the appropriate span is emitted to `sess` and a dummy value is returned. -/// -/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator. -/// In the 2018 edition however, `?` is a Kleene operator, and not a separator. -fn parse_sep_and_kleene_op<I>( - input: &mut Peekable<I>, +fn parse_sep_and_kleene_op( + input: &mut Peekable<impl Iterator<Item = tokenstream::TokenTree>>, span: Span, sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - edition: Edition, - macro_node_id: NodeId, -) -> (Option<token::Token>, KleeneOp) -where - I: Iterator<Item = tokenstream::TokenTree>, -{ - match edition { - Edition::Edition2015 => parse_sep_and_kleene_op_2015( - input, - span, - sess, - features, - attrs, - macro_node_id, - ), - Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs), - } -} - -// `?` is a separator (with a migration warning) and never a KleeneOp. -fn parse_sep_and_kleene_op_2015<I>( - input: &mut Peekable<I>, - span: Span, - sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - macro_node_id: NodeId, -) -> (Option<token::Token>, KleeneOp) -where - I: Iterator<Item = tokenstream::TokenTree>, -{ +) -> (Option<Token>, KleeneToken) { // We basically look at two token trees here, denoted as #1 and #2 below let span = match parse_kleene_op(input, span) { - // #1 is a `+` or `*` KleeneOp - // - // `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so - // we need to look ahead one more token to be sure. - Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op), - - // #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator - // or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1 - // token to find out which. - Ok(Ok((op, op1_span))) => { - assert_eq!(op, KleeneOp::ZeroOrOne); - - // Lookahead at #2. If it is a KleenOp, then #1 is a separator. - let is_1_sep = if let Some(&tokenstream::TokenTree::Token(_, ref tok2)) = input.peek() { - kleene_op(tok2).is_some() - } else { - false - }; - - if is_1_sep { - // #1 is a separator and #2 should be a KleepeOp. - // (N.B. We need to advance the input iterator.) - match parse_kleene_op(input, span) { - // #2 is `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition. - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a Kleene op, which is the only valid option - Ok(Ok((op, _))) => { - // Warn that `?` as a separator will be deprecated - sess.buffer_lint( - BufferedEarlyLintId::QuestionMarkMacroSep, - op1_span, - macro_node_id, - "using `?` as a separator is deprecated and will be \ - a hard error in an upcoming edition", - ); - - return (Some(token::Question), op); - } - - // #2 is a random token (this is an error) :( - Ok(Err((_, _))) => op1_span, - - // #2 is not even a token at all :( - Err(_) => op1_span, - } - } else { - // `?` is not allowed as a Kleene op in 2015, - // but is allowed in the 2018 edition - sess.span_diagnostic - .struct_span_err(op1_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - } + // #1 is a `?`, `+`, or `*` KleeneOp + Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), // #1 is a separator followed by #2, a KleeneOp - Ok(Err((tok, span))) => match parse_kleene_op(input, span) { - // #2 is a `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, _))) => return (Some(tok), op), - - // #2 is a random token :( - Ok(Err((_, span))) => span, - - // #2 is not a token at all :( - Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - sess.span_diagnostic.span_err(span, "expected `*` or `+`"); - - // Return a dummy - (None, KleeneOp::ZeroOrMore) -} - -// `?` is a Kleene op, not a separator -fn parse_sep_and_kleene_op_2018<I>( - input: &mut Peekable<I>, - span: Span, - sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], -) -> (Option<token::Token>, KleeneOp) -where - I: Iterator<Item = tokenstream::TokenTree>, -{ - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `?` (needs feature gate) - Ok(Ok((op, _op1_span))) if op == KleeneOp::ZeroOrOne => { - return (None, op); - } - - // #1 is a `+` or `*` KleeneOp - Ok(Ok((op, _))) => return (None, op), - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err((tok, span))) => match parse_kleene_op(input, span) { + Ok(Err(token)) => match parse_kleene_op(input, token.span) { // #2 is the `?` Kleene op, which does not take a separator (error) - Ok(Ok((op, _op2_span))) if op == KleeneOp::ZeroOrOne => { + Ok(Ok((KleeneOp::ZeroOrOne, span))) => { // Error! sess.span_diagnostic.span_err( - span, + token.span, "the `?` macro repetition operator does not take a separator", ); // Return a dummy - return (None, KleeneOp::ZeroOrMore); + return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); } // #2 is a KleeneOp :D - Ok(Ok((op, _))) => return (Some(tok), op), - - // #2 is a random token :( - Ok(Err((_, span))) => span, + Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), - // #2 is not a token at all :( - Err(span) => span, + // #2 is a random token or not a token at all :( + Ok(Err(Token { span, .. })) | Err(span) => span, }, // #1 is not a token @@ -607,9 +439,8 @@ where }; // If we ever get to this point, we have experienced an "unexpected token" error - sess.span_diagnostic - .span_err(span, "expected one of: `*`, `+`, or `?`"); + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); // Return a dummy - (None, KleeneOp::ZeroOrMore) + (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) } diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index bd2adb5ac13..214e721fd15 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -1,39 +1,29 @@ use crate::ast::Ident; use crate::ext::base::ExtCtxt; use crate::ext::expand::Marker; -use crate::ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; +use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use crate::ext::tt::quoted; use crate::mut_visit::noop_visit_tt; -use crate::parse::token::{self, Token, NtTT}; +use crate::parse::token::{self, NtTT, Token}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use smallvec::{smallvec, SmallVec}; -use syntax_pos::DUMMY_SP; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; use std::mem; -use std::ops::Add; -use std::rc::Rc; -// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). +/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`). enum Frame { - Delimited { - forest: Lrc<quoted::Delimited>, - idx: usize, - span: DelimSpan, - }, - Sequence { - forest: Lrc<quoted::SequenceRepetition>, - idx: usize, - sep: Option<Token>, - }, + Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan }, + Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> }, } impl Frame { + /// Construct a new frame around the delimited set of tokens. fn new(tts: Vec<quoted::TokenTree>) -> Frame { - let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts: tts }); - Frame::Delimited { forest: forest, idx: 0, span: DelimSpan::dummy() } + let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts }); + Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() } } } @@ -54,84 +44,158 @@ impl Iterator for Frame { } } -/// This can do Macro-By-Example transcription. On the other hand, if -/// `src` contains no `TokenTree::{Sequence, MetaVar, MetaVarDecl}`s, `interp` can -/// (and should) be None. -pub fn transcribe(cx: &ExtCtxt<'_>, - interp: Option<FxHashMap<Ident, Rc<NamedMatch>>>, - src: Vec<quoted::TokenTree>) - -> TokenStream { +/// This can do Macro-By-Example transcription. +/// - `interp` is a map of meta-variables to the tokens (non-terminals) they matched in the +/// invocation. We are assuming we already know there is a match. +/// - `src` is the RHS of the MBE, that is, the "example" we are filling in. +/// +/// For example, +/// +/// ```rust +/// macro_rules! foo { +/// ($id:ident) => { println!("{}", stringify!($id)); } +/// } +/// +/// foo!(bar); +/// ``` +/// +/// `interp` would contain `$id => bar` and `src` would contain `println!("{}", stringify!($id));`. +/// +/// `transcribe` would return a `TokenStream` containing `println!("{}", stringify!(bar));`. +/// +/// Along the way, we do some additional error checking. +pub(super) fn transcribe( + cx: &ExtCtxt<'_>, + interp: &FxHashMap<Ident, NamedMatch>, + src: Vec<quoted::TokenTree>, +) -> TokenStream { + // Nothing for us to transcribe... + if src.is_empty() { + return TokenStream::empty(); + } + + // We descend into the RHS (`src`), expanding things as we go. This stack contains the things + // we have yet to expand/are still expanding. We start the stack off with the whole RHS. let mut stack: SmallVec<[Frame; 1]> = smallvec![Frame::new(src)]; - let interpolations = interp.unwrap_or_else(FxHashMap::default); /* just a convenience */ + + // As we descend in the RHS, we will need to be able to match nested sequences of matchers. + // `repeats` keeps track of where we are in matching at each level, with the last element being + // the most deeply nested sequence. This is used as a stack. let mut repeats = Vec::new(); + + // `result` contains resulting token stream from the TokenTree we just finished processing. At + // the end, this will contain the full result of transcription, but at arbitrary points during + // `transcribe`, `result` will contain subsets of the final result. + // + // Specifically, as we descend into each TokenTree, we will push the existing results onto the + // `result_stack` and clear `results`. We will then produce the results of transcribing the + // TokenTree into `results`. Then, as we unwind back out of the `TokenTree`, we will pop the + // `result_stack` and append `results` too it to produce the new `results` up to that point. + // + // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level + // again, and we are done transcribing. let mut result: Vec<TreeAndJoint> = Vec::new(); let mut result_stack = Vec::new(); loop { + // Look at the last frame on the stack. let tree = if let Some(tree) = stack.last_mut().unwrap().next() { + // If it still has a TokenTree we have not looked at yet, use that tree. tree - } else { - if let Frame::Sequence { ref mut idx, ref sep, .. } = *stack.last_mut().unwrap() { - let (ref mut repeat_idx, repeat_len) = *repeats.last_mut().unwrap(); + } + // The else-case never produces a value for `tree` (it `continue`s or `return`s). + else { + // Otherwise, if we have just reached the end of a sequence and we can keep repeating, + // go back to the beginning of the sequence. + if let Frame::Sequence { idx, sep, .. } = stack.last_mut().unwrap() { + let (repeat_idx, repeat_len) = repeats.last_mut().unwrap(); *repeat_idx += 1; - if *repeat_idx < repeat_len { + if repeat_idx < repeat_len { *idx = 0; - if let Some(sep) = sep.clone() { - // repeat same span, I guess - let prev_span = match result.last() { - Some((tt, _)) => tt.span(), - None => DUMMY_SP, - }; - result.push(TokenTree::Token(prev_span, sep).into()); + if let Some(sep) = sep { + result.push(TokenTree::Token(sep.clone()).into()); } - continue + continue; } } + // We are done with the top of the stack. Pop it. Depending on what it was, we do + // different things. Note that the outermost item must be the delimited, wrapped RHS + // that was passed in originally to `transcribe`. match stack.pop().unwrap() { + // Done with a sequence. Pop from repeats. Frame::Sequence { .. } => { repeats.pop(); } + + // We are done processing a Delimited. If this is the top-level delimited, we are + // done. Otherwise, we unwind the result_stack to append what we have produced to + // any previous results. Frame::Delimited { forest, span, .. } => { if result_stack.is_empty() { + // No results left to compute! We are back at the top-level. return TokenStream::new(result); } - let tree = TokenTree::Delimited( - span, - forest.delim, - TokenStream::new(result).into(), - ); + + // Step back into the parent Delimited. + let tree = + TokenTree::Delimited(span, forest.delim, TokenStream::new(result).into()); result = result_stack.pop().unwrap(); result.push(tree.into()); } } - continue + continue; }; + // At this point, we know we are in the middle of a TokenTree (the last one on `stack`). + // `tree` contains the next `TokenTree` to be processed. match tree { - quoted::TokenTree::Sequence(sp, seq) => { - // FIXME(pcwalton): Bad copy. - match lockstep_iter_size("ed::TokenTree::Sequence(sp, seq.clone()), - &interpolations, - &repeats) { + // We are descending into a sequence. We first make sure that the matchers in the RHS + // and the matches in `interp` have the same shape. Otherwise, either the caller or the + // macro writer has made a mistake. + seq @ quoted::TokenTree::Sequence(..) => { + match lockstep_iter_size(&seq, interp, &repeats) { LockstepIterSize::Unconstrained => { - cx.span_fatal(sp.entire(), /* blame macro writer */ - "attempted to repeat an expression \ - containing no syntax \ - variables matched as repeating at this depth"); + cx.span_fatal( + seq.span(), /* blame macro writer */ + "attempted to repeat an expression containing no syntax variables \ + matched as repeating at this depth", + ); } + LockstepIterSize::Contradiction(ref msg) => { - // FIXME #2887 blame macro invoker instead - cx.span_fatal(sp.entire(), &msg[..]); + // FIXME: this really ought to be caught at macro definition time... It + // happens when two meta-variables are used in the same repetition in a + // sequence, but they come from different sequence matchers and repeat + // different amounts. + cx.span_fatal(seq.span(), &msg[..]); } + LockstepIterSize::Constraint(len, _) => { + // We do this to avoid an extra clone above. We know that this is a + // sequence already. + let (sp, seq) = if let quoted::TokenTree::Sequence(sp, seq) = seq { + (sp, seq) + } else { + unreachable!() + }; + + // Is the repetition empty? if len == 0 { - if seq.op == quoted::KleeneOp::OneOrMore { - // FIXME #2887 blame invoker + if seq.kleene.op == quoted::KleeneOp::OneOrMore { + // FIXME: this really ought to be caught at macro definition + // time... It happens when the Kleene operator in the matcher and + // the body for the same meta-variable do not match. cx.span_fatal(sp.entire(), "this must repeat at least once"); } } else { + // 0 is the initial counter (we have done 0 repretitions so far). `len` + // is the total number of reptitions we should generate. repeats.push((0, len)); + + // The first time we encounter the sequence we push it to the stack. It + // then gets reused (see the beginning of the loop) until we are done + // repeating. stack.push(Frame::Sequence { idx: 0, sep: seq.separator.clone(), @@ -141,56 +205,84 @@ pub fn transcribe(cx: &ExtCtxt<'_>, } } } - // FIXME #2887: think about span stuff here + + // Replace the meta-var with the matched token tree from the invocation. quoted::TokenTree::MetaVar(mut sp, ident) => { - if let Some(cur_matched) = lookup_cur_matched(ident, &interpolations, &repeats) { - if let MatchedNonterminal(ref nt) = *cur_matched { + // Find the matched nonterminal from the macro invocation, and use it to replace + // the meta-var. + if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + if let MatchedNonterminal(ref nt) = cur_matched { + // FIXME #2887: why do we apply a mark when matching a token tree meta-var + // (e.g. `$x:tt`), but not when we are matching any other type of token + // tree? if let NtTT(ref tt) = **nt { result.push(tt.clone().into()); } else { - sp = sp.apply_mark(cx.current_expansion.mark); - let token = TokenTree::Token(sp, Token::Interpolated(nt.clone())); + sp = sp.apply_mark(cx.current_expansion.id); + let token = TokenTree::token(token::Interpolated(nt.clone()), sp); result.push(token.into()); } } else { - cx.span_fatal(sp, /* blame the macro writer */ - &format!("variable '{}' is still repeating at this depth", ident)); + // We were unable to descend far enough. This is an error. + cx.span_fatal( + sp, /* blame the macro writer */ + &format!("variable '{}' is still repeating at this depth", ident), + ); } } else { + // If we aren't able to match the meta-var, we push it back into the result but + // with modified syntax context. (I believe this supports nested macros). let ident = - Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark)); - sp = sp.apply_mark(cx.current_expansion.mark); - result.push(TokenTree::Token(sp, token::Dollar).into()); - result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into()); + Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.id)); + sp = sp.apply_mark(cx.current_expansion.id); + result.push(TokenTree::token(token::Dollar, sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); } } + + // If we are entering a new delimiter, we push its contents to the `stack` to be + // processed, and we push all of the currently produced results to the `result_stack`. + // We will produce all of the results of the inside of the `Delimited` and then we will + // jump back out of the Delimited, pop the result_stack and add the new results back to + // the previous results (from outside the Delimited). quoted::TokenTree::Delimited(mut span, delimited) => { - span = span.apply_mark(cx.current_expansion.mark); - stack.push(Frame::Delimited { forest: delimited, idx: 0, span: span }); - result_stack.push(mem::replace(&mut result, Vec::new())); + span = span.apply_mark(cx.current_expansion.id); + stack.push(Frame::Delimited { forest: delimited, idx: 0, span }); + result_stack.push(mem::take(&mut result)); } - quoted::TokenTree::Token(sp, tok) => { - let mut marker = Marker(cx.current_expansion.mark); - let mut tt = TokenTree::Token(sp, tok); + + // Nothing much to do here. Just push the token to the result, being careful to + // preserve syntax context. + quoted::TokenTree::Token(token) => { + let mut marker = Marker(cx.current_expansion.id); + let mut tt = TokenTree::Token(token); noop_visit_tt(&mut tt, &mut marker); result.push(tt.into()); } + + // There should be no meta-var declarations in the invocation of a macro. quoted::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"), } } } -fn lookup_cur_matched(ident: Ident, - interpolations: &FxHashMap<Ident, Rc<NamedMatch>>, - repeats: &[(usize, usize)]) - -> Option<Rc<NamedMatch>> { +/// Lookup the meta-var named `ident` and return the matched token tree from the invocation using +/// the set of matches `interpolations`. +/// +/// See the definition of `repeats` in the `transcribe` function. `repeats` is used to descend +/// into the right place in nested matchers. If we attempt to descend too far, the macro writer has +/// made a mistake, and we return `None`. +fn lookup_cur_matched<'a>( + ident: Ident, + interpolations: &'a FxHashMap<Ident, NamedMatch>, + repeats: &[(usize, usize)], +) -> Option<&'a NamedMatch> { interpolations.get(&ident).map(|matched| { - let mut matched = matched.clone(); + let mut matched = matched; for &(idx, _) in repeats { - let m = matched.clone(); - match *m { + match matched { MatchedNonterminal(_) => break, - MatchedSeq(ref ads, _) => matched = Rc::new(ads[idx].clone()), + MatchedSeq(ref ads, _) => matched = ads.get(idx).unwrap(), } } @@ -198,17 +290,30 @@ fn lookup_cur_matched(ident: Ident, }) } +/// An accumulator over a TokenTree to be used with `fold`. During transcription, we need to make +/// sure that the size of each sequence and all of its nested sequences are the same as the sizes +/// of all the matched (nested) sequences in the macro invocation. If they don't match, somebody +/// has made a mistake (either the macro writer or caller). #[derive(Clone)] enum LockstepIterSize { + /// No constraints on length of matcher. This is true for any TokenTree variants except a + /// `MetaVar` with an actual `MatchedSeq` (as opposed to a `MatchedNonterminal`). Unconstrained, + + /// A `MetaVar` with an actual `MatchedSeq`. The length of the match and the name of the + /// meta-var are returned. Constraint(usize, Ident), + + /// Two `Constraint`s on the same sequence had different lengths. This is an error. Contradiction(String), } -impl Add for LockstepIterSize { - type Output = LockstepIterSize; - - fn add(self, other: LockstepIterSize) -> LockstepIterSize { +impl LockstepIterSize { + /// Find incompatibilities in matcher/invocation sizes. + /// - `Unconstrained` is compatible with everything. + /// - `Contradiction` is incompatible with everything. + /// - `Constraint(len)` is only compatible with other constraints of the same length. + fn with(self, other: LockstepIterSize) -> LockstepIterSize { match self { LockstepIterSize::Unconstrained => other, LockstepIterSize::Contradiction(_) => self, @@ -217,9 +322,10 @@ impl Add for LockstepIterSize { LockstepIterSize::Contradiction(_) => other, LockstepIterSize::Constraint(r_len, _) if l_len == r_len => self, LockstepIterSize::Constraint(r_len, r_id) => { - let msg = format!("inconsistent lockstep iteration: \ - '{}' has {} items, but '{}' has {}", - l_id, l_len, r_id, r_len); + let msg = format!( + "meta-variable `{}` repeats {} times, but `{}` repeats {} times", + l_id, l_len, r_id, r_len + ); LockstepIterSize::Contradiction(msg) } }, @@ -227,30 +333,38 @@ impl Add for LockstepIterSize { } } -fn lockstep_iter_size(tree: "ed::TokenTree, - interpolations: &FxHashMap<Ident, Rc<NamedMatch>>, - repeats: &[(usize, usize)]) - -> LockstepIterSize { +/// Given a `tree`, make sure that all sequences have the same length as the matches for the +/// appropriate meta-vars in `interpolations`. +/// +/// Note that if `repeats` does not match the exact correct depth of a meta-var, +/// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of +/// multiple nested matcher sequences. +fn lockstep_iter_size( + tree: "ed::TokenTree, + interpolations: &FxHashMap<Ident, NamedMatch>, + repeats: &[(usize, usize)], +) -> LockstepIterSize { use quoted::TokenTree; match *tree { TokenTree::Delimited(_, ref delimed) => { delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size + lockstep_iter_size(tt, interpolations, repeats) + size.with(lockstep_iter_size(tt, interpolations, repeats)) }) - }, + } TokenTree::Sequence(_, ref seq) => { seq.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| { - size + lockstep_iter_size(tt, interpolations, repeats) + size.with(lockstep_iter_size(tt, interpolations, repeats)) }) - }, - TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => + } + TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => { match lookup_cur_matched(name, interpolations, repeats) { - Some(matched) => match *matched { + Some(matched) => match matched { MatchedNonterminal(_) => LockstepIterSize::Unconstrained, MatchedSeq(ref ads, _) => LockstepIterSize::Constraint(ads.len(), name), }, - _ => LockstepIterSize::Unconstrained - }, + _ => LockstepIterSize::Unconstrained, + } + } TokenTree::Token(..) => LockstepIterSize::Unconstrained, } } diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 279e2089f5d..08a113b53d0 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -15,21 +15,26 @@ use AttributeType::*; use AttributeGate::*; -use crate::ast::{self, NodeId, GenericParam, GenericParamKind, PatKind, RangeEnd}; -use crate::attr; -use crate::early_buffered_lints::BufferedEarlyLintId; +use crate::ast::{ + self, AssocTyConstraint, AssocTyConstraintKind, NodeId, GenericParam, GenericParamKind, + PatKind, RangeEnd, +}; +use crate::attr::{self, check_builtin_attribute, AttributeTemplate}; use crate::source_map::Spanned; use crate::edition::{ALL_EDITIONS, Edition}; use crate::visit::{self, FnKind, Visitor}; use crate::parse::{token, ParseSess}; -use crate::symbol::Symbol; +use crate::parse::parser::Parser; +use crate::symbol::{Symbol, sym}; use crate::tokenstream::TokenTree; -use errors::{DiagnosticBuilder, Handler}; +use errors::{Applicability, DiagnosticBuilder, Handler}; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lock; use rustc_target::spec::abi::Abi; -use syntax_pos::{Span, DUMMY_SP}; +use syntax_pos::{Span, DUMMY_SP, MultiSpan}; use log::debug; +use lazy_static::lazy_static; use std::env; @@ -47,8 +52,8 @@ macro_rules! declare_features { /// Represents active features that are currently being implemented or /// currently being considered for addition/removal. const ACTIVE_FEATURES: - &[(&str, &str, Option<u32>, Option<Edition>, fn(&mut Features, Span))] = - &[$((stringify!($feature), $ver, $issue, $edition, set!($feature))),+]; + &[(Symbol, &str, Option<u32>, Option<Edition>, fn(&mut Features, Span))] = + &[$((sym::$feature, $ver, $issue, $edition, set!($feature))),+]; /// A set of features to be used by later passes. #[derive(Clone)] @@ -79,22 +84,22 @@ macro_rules! declare_features { ($((removed, $feature: ident, $ver: expr, $issue: expr, None, $reason: expr),)+) => { /// Represents unstable features which have since been removed (it was once Active) - const REMOVED_FEATURES: &[(&str, &str, Option<u32>, Option<&str>)] = &[ - $((stringify!($feature), $ver, $issue, $reason)),+ + const REMOVED_FEATURES: &[(Symbol, &str, Option<u32>, Option<&str>)] = &[ + $((sym::$feature, $ver, $issue, $reason)),+ ]; }; ($((stable_removed, $feature: ident, $ver: expr, $issue: expr, None),)+) => { /// Represents stable features which have since been removed (it was once Accepted) - const STABLE_REMOVED_FEATURES: &[(&str, &str, Option<u32>, Option<&str>)] = &[ - $((stringify!($feature), $ver, $issue, None)),+ + const STABLE_REMOVED_FEATURES: &[(Symbol, &str, Option<u32>, Option<&str>)] = &[ + $((sym::$feature, $ver, $issue, None)),+ ]; }; ($((accepted, $feature: ident, $ver: expr, $issue: expr, None),)+) => { /// Those language feature has since been Accepted (it was once Active) - const ACCEPTED_FEATURES: &[(&str, &str, Option<u32>, Option<&str>)] = &[ - $((stringify!($feature), $ver, $issue, None)),+ + const ACCEPTED_FEATURES: &[(Symbol, &str, Option<u32>, Option<&str>)] = &[ + $((sym::$feature, $ver, $issue, None)),+ ]; } } @@ -107,124 +112,216 @@ macro_rules! declare_features { // was set. This is most important for knowing when a particular feature became // stable (active). // +// Note that the features are grouped into internal/user-facing and then +// sorted by version inside those groups. This is inforced with tidy. +// // N.B., `tools/tidy/src/features.rs` parses this information directly out of the // source, so take care when modifying it. declare_features! ( - (active, asm, "1.0.0", Some(29722), None), - (active, concat_idents, "1.0.0", Some(29599), None), - (active, link_args, "1.0.0", Some(29596), None), - (active, log_syntax, "1.0.0", Some(29598), None), - (active, non_ascii_idents, "1.0.0", Some(55467), None), - (active, plugin_registrar, "1.0.0", Some(29597), None), - (active, thread_local, "1.0.0", Some(29594), None), - (active, trace_macros, "1.0.0", Some(29598), None), + // ------------------------------------------------------------------------- + // feature-group-start: internal feature gates + // ------------------------------------------------------------------------- + + // no-tracking-issue-start - // rustc internal, for now + // Allows using compiler's own crates. + (active, rustc_private, "1.0.0", Some(27812), None), + + // Allows using the `rust-intrinsic`'s "ABI". (active, intrinsics, "1.0.0", None, None), + + // Allows using `#[lang = ".."]` attribute for linking items to special compiler logic. (active, lang_items, "1.0.0", None, None), - (active, format_args_nl, "1.29.0", None, None), - (active, link_llvm_intrinsics, "1.0.0", Some(29602), None), - (active, linkage, "1.0.0", Some(29603), None), + // Allows using the `#[stable]` and `#[unstable]` attributes. + (active, staged_api, "1.0.0", None, None), - // rustc internal + // Allows using `#[allow_internal_unstable]`. This is an + // attribute on `macro_rules!` and can't use the attribute handling + // below (it has to be checked before expansion possibly makes + // macros disappear). + (active, allow_internal_unstable, "1.0.0", None, None), + + // Allows using `#[allow_internal_unsafe]`. This is an + // attribute on `macro_rules!` and can't use the attribute handling + // below (it has to be checked before expansion possibly makes + // macros disappear). + (active, allow_internal_unsafe, "1.0.0", None, None), + + // Allows using the macros: + // + `__diagnostic_used` + // + `__register_diagnostic` + // +`__build_diagnostic_array` (active, rustc_diagnostic_macros, "1.0.0", None, None), + + // Allows using `#[rustc_const_unstable(feature = "foo", ..)]` which + // lets a function to be `const` when opted into with `#![feature(foo)]`. (active, rustc_const_unstable, "1.0.0", None, None), + + // no-tracking-issue-end + + // Allows using `#[link_name="llvm.*"]`. + (active, link_llvm_intrinsics, "1.0.0", Some(29602), None), + + // Allows using `rustc_*` attributes (RFC 572). + (active, rustc_attrs, "1.0.0", Some(29642), None), + + // Allows using `#[on_unimplemented(..)]` on traits. + (active, on_unimplemented, "1.0.0", Some(29628), None), + + // Allows using the `box $expr` syntax. (active, box_syntax, "1.0.0", Some(49733), None), - (active, unboxed_closures, "1.0.0", Some(29625), None), - (active, fundamental, "1.0.0", Some(29635), None), + // Allows using `#[main]` to replace the entrypoint `#[lang = "start"]` calls. (active, main, "1.0.0", Some(29634), None), - (active, needs_allocator, "1.4.0", Some(27389), None), - (active, on_unimplemented, "1.0.0", Some(29628), None), - (active, plugin, "1.0.0", Some(29597), None), - (active, simd_ffi, "1.0.0", Some(27731), None), + + // Allows using `#[start]` on a function indicating that it is the program entrypoint. (active, start, "1.0.0", Some(29633), None), - (active, structural_match, "1.8.0", Some(31434), None), - (active, panic_runtime, "1.10.0", Some(32837), None), - (active, needs_panic_runtime, "1.10.0", Some(32837), None), - // Features specific to OIBIT (auto traits) - (active, optin_builtin_traits, "1.0.0", Some(13231), None), + // Allows using the `#[fundamental]` attribute. + (active, fundamental, "1.0.0", Some(29635), None), - // Allows `#[staged_api]`. - // - // rustc internal - (active, staged_api, "1.0.0", None, None), + // Allows using the `rust-call` ABI. + (active, unboxed_closures, "1.0.0", Some(29625), None), - // Allows `#![no_core]`. - (active, no_core, "1.3.0", Some(29639), None), + // Allows using the `#[linkage = ".."]` attribute. + (active, linkage, "1.0.0", Some(29603), None), - // Allows the use of `box` in patterns (RFC 469). + // Allows features specific to OIBIT (auto traits). + (active, optin_builtin_traits, "1.0.0", Some(13231), None), + + // Allows using `box` in patterns (RFC 469). (active, box_patterns, "1.0.0", Some(29641), None), - // Allows the use of the `unsafe_destructor_blind_to_params` attribute (RFC 1238). - (active, dropck_parametricity, "1.3.0", Some(28498), None), + // no-tracking-issue-start + + // Allows using `#[prelude_import]` on glob `use` items. + (active, prelude_import, "1.2.0", None, None), + + // no-tracking-issue-end + + // no-tracking-issue-start + + // Allows using `#[omit_gdb_pretty_printer_section]`. + (active, omit_gdb_pretty_printer_section, "1.5.0", None, None), + + // Allows using the `vectorcall` ABI. + (active, abi_vectorcall, "1.7.0", None, None), + + // no-tracking-issue-end + + // Allows using `#[structural_match]` which indicates that a type is structurally matchable. + (active, structural_match, "1.8.0", Some(31434), None), // Allows using the `may_dangle` attribute (RFC 1327). (active, dropck_eyepatch, "1.10.0", Some(34761), None), - // Allows the use of custom attributes (RFC 572). - (active, custom_attribute, "1.0.0", Some(29642), None), + // Allows using the `#![panic_runtime]` attribute. + (active, panic_runtime, "1.10.0", Some(32837), None), - // Allows the use of `rustc_*` attributes (RFC 572). - (active, rustc_attrs, "1.0.0", Some(29642), None), + // Allows declaring with `#![needs_panic_runtime]` that a panic runtime is needed. + (active, needs_panic_runtime, "1.10.0", Some(32837), None), - // Allows the use of non lexical lifetimes (RFC 2094). - (active, nll, "1.0.0", Some(43234), None), + // no-tracking-issue-start - // Allows the use of `#[allow_internal_unstable]`. This is an - // attribute on `macro_rules!` and can't use the attribute handling - // below (it has to be checked before expansion possibly makes - // macros disappear). - // - // rustc internal - (active, allow_internal_unstable, "1.0.0", None, None), + // Allows identifying the `compiler_builtins` crate. + (active, compiler_builtins, "1.13.0", None, None), - // Allows the use of `#[allow_internal_unsafe]`. This is an - // attribute on `macro_rules!` and can't use the attribute handling - // below (it has to be checked before expansion possibly makes - // macros disappear). - // - // rustc internal - (active, allow_internal_unsafe, "1.0.0", None, None), + // Allows using the `unadjusted` ABI; perma-unstable. + (active, abi_unadjusted, "1.16.0", None, None), - // Allows the use of slice patterns (issue #23121). - (active, slice_patterns, "1.0.0", Some(23121), None), + // Allows identifying crates that contain sanitizer runtimes. + (active, sanitizer_runtime, "1.17.0", None, None), - // Allows the definition of `const` functions with some advanced features. - (active, const_fn, "1.2.0", Some(57563), None), + // Used to identify crates that contain the profiler runtime. + (active, profiler_runtime, "1.18.0", None, None), - // Allows accessing fields of unions inside `const` functions. - (active, const_fn_union, "1.27.0", Some(51909), None), + // Allows using the `thiscall` ABI. + (active, abi_thiscall, "1.19.0", None, None), - // Allows casting raw pointers to `usize` during const eval. - (active, const_raw_ptr_to_usize_cast, "1.27.0", Some(51910), None), + // Allows using `#![needs_allocator]`, an implementation detail of `#[global_allocator]`. + (active, allocator_internals, "1.20.0", None, None), - // Allows dereferencing raw pointers during const eval. - (active, const_raw_ptr_deref, "1.27.0", Some(51911), None), + // no-tracking-issue-end - // Allows reinterpretation of the bits of a value of one type as another type during const eval. - (active, const_transmute, "1.29.0", Some(53605), None), + // Added for testing E0705; perma-unstable. + (active, test_2018_feature, "1.31.0", Some(0), Some(Edition::Edition2018)), - // Allows comparing raw pointers during const eval. - (active, const_compare_raw_pointers, "1.27.0", Some(53020), None), + // ------------------------------------------------------------------------- + // feature-group-end: internal feature gates + // ------------------------------------------------------------------------- - // Allows panicking during const eval (producing compile-time errors). - (active, const_panic, "1.30.0", Some(51999), None), + // ------------------------------------------------------------------------- + // feature-group-start: actual feature gates (target features) + // ------------------------------------------------------------------------- - // Allows using `#[prelude_import]` on glob `use` items. - // - // rustc internal - (active, prelude_import, "1.2.0", None, None), + // FIXME: Document these and merge with the list below. - // Allows default type parameters to influence type inference. - (active, default_type_parameter_fallback, "1.3.0", Some(27336), None), + // Unstable `#[target_feature]` directives. + (active, arm_target_feature, "1.27.0", Some(44839), None), + (active, aarch64_target_feature, "1.27.0", Some(44839), None), + (active, hexagon_target_feature, "1.27.0", Some(44839), None), + (active, powerpc_target_feature, "1.27.0", Some(44839), None), + (active, mips_target_feature, "1.27.0", Some(44839), None), + (active, avx512_target_feature, "1.27.0", Some(44839), None), + (active, mmx_target_feature, "1.27.0", Some(44839), None), + (active, sse4a_target_feature, "1.27.0", Some(44839), None), + (active, tbm_target_feature, "1.27.0", Some(44839), None), + (active, wasm_target_feature, "1.30.0", Some(44839), None), + (active, adx_target_feature, "1.32.0", Some(44839), None), + (active, cmpxchg16b_target_feature, "1.32.0", Some(44839), None), + (active, movbe_target_feature, "1.34.0", Some(44839), None), + (active, rtm_target_feature, "1.35.0", Some(44839), None), + (active, f16c_target_feature, "1.36.0", Some(44839), None), + + // ------------------------------------------------------------------------- + // feature-group-end: actual feature gates (target features) + // ------------------------------------------------------------------------- + + // ------------------------------------------------------------------------- + // feature-group-start: actual feature gates + // ------------------------------------------------------------------------- + + // Allows using the `#[link_args]` attribute. + (active, link_args, "1.0.0", Some(29596), None), + + // Allows defining identifiers beyond ASCII. + (active, non_ascii_idents, "1.0.0", Some(55467), None), + + // Allows using `#[plugin_registrar]` on functions. + (active, plugin_registrar, "1.0.0", Some(29597), None), + + // Allows using `#![plugin(myplugin)]`. + (active, plugin, "1.0.0", Some(29597), None), + + // Allows using `#[thread_local]` on `static` items. + (active, thread_local, "1.0.0", Some(29594), None), + + // Allows the use of SIMD types in functions declared in `extern` blocks. + (active, simd_ffi, "1.0.0", Some(27731), None), + + // Allows using custom attributes (RFC 572). + (active, custom_attribute, "1.0.0", Some(29642), None), + + // Allows using non lexical lifetimes (RFC 2094). + (active, nll, "1.0.0", Some(43234), None), + + // Allows using slice patterns. + (active, slice_patterns, "1.0.0", Some(62254), None), + + // Allows the definition of `const` functions with some advanced features. + (active, const_fn, "1.2.0", Some(57563), None), // Allows associated type defaults. (active, associated_type_defaults, "1.2.0", Some(29661), None), + // Allows `#![no_core]`. + (active, no_core, "1.3.0", Some(29639), None), + + // Allows default type parameters to influence type inference. + (active, default_type_parameter_fallback, "1.3.0", Some(27336), None), + // Allows `repr(simd)` and importing the various simd intrinsics. (active, repr_simd, "1.4.0", Some(27731), None), @@ -236,17 +333,9 @@ declare_features! ( // Permits specifying whether a function should permit unwinding or abort on unwind. (active, unwind_attributes, "1.4.0", Some(58760), None), - // Allows the use of `#[naked]` on functions. - (active, naked_functions, "1.9.0", Some(32408), None), - // Allows `#[no_debug]`. (active, no_debug, "1.5.0", Some(29721), None), - // Allows `#[omit_gdb_pretty_printer_section]`. - // - // rustc internal - (active, omit_gdb_pretty_printer_section, "1.5.0", None, None), - // Allows attributes on expressions and non-item statements. (active, stmt_expr_attributes, "1.6.0", Some(15701), None), @@ -256,19 +345,19 @@ declare_features! ( // Allows `cfg(target_thread_local)`. (active, cfg_target_thread_local, "1.7.0", Some(29594), None), - // rustc internal - (active, abi_vectorcall, "1.7.0", None, None), - - // Allows `X..Y` patterns. - (active, exclusive_range_pattern, "1.11.0", Some(37854), None), - - // impl specialization (RFC 1210) + // Allows specialization of implementations (RFC 1210). (active, specialization, "1.7.0", Some(31844), None), + // Allows using `#[naked]` on functions. + (active, naked_functions, "1.9.0", Some(32408), None), + // Allows `cfg(target_has_atomic = "...")`. (active, cfg_target_has_atomic, "1.9.0", Some(32976), None), - // The `!` type. Does not imply 'exhaustive_patterns' (below) any more. + // Allows `X..Y` patterns. + (active, exclusive_range_pattern, "1.11.0", Some(37854), None), + + // Allows the `!` type. Does not imply 'exhaustive_patterns' (below) any more. (active, never_type, "1.13.0", Some(35121), None), // Allows exhaustive pattern matching on types that contain uninhabited types. @@ -277,206 +366,220 @@ declare_features! ( // Allows untagged unions `union U { ... }`. (active, untagged_unions, "1.13.0", Some(32836), None), - // Used to identify the `compiler_builtins` crate. - // - // rustc internal. - (active, compiler_builtins, "1.13.0", None, None), - // Allows `#[link(..., cfg(..))]`. (active, link_cfg, "1.14.0", Some(37406), None), // Allows `extern "ptx-*" fn()`. (active, abi_ptx, "1.15.0", Some(38788), None), - // The `repr(i128)` annotation for enums. + // Allows the `#[repr(i128)]` attribute for enums. (active, repr128, "1.16.0", Some(35118), None), - // Allows the use of `#[ffi_returns_twice]` on foreign functions. - (active, ffi_returns_twice, "1.34.0", Some(58314), None), - - // The `unadjusted` ABI; perma-unstable. - // - // rustc internal - (active, abi_unadjusted, "1.16.0", None, None), - - // Declarative macros 2.0 (`macro`). - (active, decl_macro, "1.17.0", Some(39412), None), - // Allows `#[link(kind="static-nobundle"...)]`. (active, static_nobundle, "1.16.0", Some(37403), None), // Allows `extern "msp430-interrupt" fn()`. (active, abi_msp430_interrupt, "1.16.0", Some(38487), None), - // Used to identify crates that contain sanitizer runtimes. - // - // rustc internal - (active, sanitizer_runtime, "1.17.0", None, None), - - // Used to identify crates that contain the profiler runtime. - // - // rustc internal - (active, profiler_runtime, "1.18.0", None, None), + // Allows declarative macros 2.0 (`macro`). + (active, decl_macro, "1.17.0", Some(39412), None), // Allows `extern "x86-interrupt" fn()`. (active, abi_x86_interrupt, "1.17.0", Some(40180), None), - // Allows the `try {...}` expression. - (active, try_blocks, "1.29.0", Some(31436), None), - - // Allows module-level inline assembly by way of `global_asm!()`. - (active, global_asm, "1.18.0", Some(35119), None), - // Allows overlapping impls of marker traits. (active, overlapping_marker_traits, "1.18.0", Some(29864), None), - // Trait attribute to allow overlapping impls. - (active, marker_trait_attr, "1.30.0", Some(29864), None), - - // rustc internal - (active, abi_thiscall, "1.19.0", None, None), - // Allows a test to fail without failing the whole suite. (active, allow_fail, "1.19.0", Some(46488), None), // Allows unsized tuple coercion. (active, unsized_tuple_coercion, "1.20.0", Some(42877), None), - // Generators + // Allows defining generators. (active, generators, "1.21.0", Some(43122), None), - // Trait aliases - (active, trait_alias, "1.24.0", Some(41517), None), - - // rustc internal - (active, allocator_internals, "1.20.0", None, None), - - // `#[doc(cfg(...))]` + // Allows `#[doc(cfg(...))]`. (active, doc_cfg, "1.21.0", Some(43781), None), - // `#[doc(masked)]` + + // Allows `#[doc(masked)]`. (active, doc_masked, "1.21.0", Some(44027), None), - // `#[doc(spotlight)]` + + // Allows `#[doc(spotlight)]`. (active, doc_spotlight, "1.22.0", Some(45040), None), - // `#[doc(include = "some-file")]` + + // Allows `#[doc(include = "some-file")]`. (active, external_doc, "1.22.0", Some(44732), None), - // Future-proofing enums/structs with `#[non_exhaustive]` attribute (RFC 2008). + // Allows future-proofing enums/structs with the `#[non_exhaustive]` attribute (RFC 2008). (active, non_exhaustive, "1.22.0", Some(44109), None), - // Adds `crate` as visibility modifier, synonymous with `pub(crate)`. + // Allows using `crate` as visibility modifier, synonymous with `pub(crate)`. (active, crate_visibility_modifier, "1.23.0", Some(53120), None), - // extern types + // Allows defining `extern type`s. (active, extern_types, "1.23.0", Some(43467), None), // Allows trait methods with arbitrary self types. (active, arbitrary_self_types, "1.23.0", Some(44874), None), - // In-band lifetime bindings (e.g., `fn foo(x: &'a u8) -> &'a u8`). + // Allows in-band quantification of lifetime bindings (e.g., `fn foo(x: &'a u8) -> &'a u8`). (active, in_band_lifetimes, "1.23.0", Some(44524), None), - // Generic associated types (RFC 1598) + // Allows associated types to be generic, e.g., `type Foo<T>;` (RFC 1598). (active, generic_associated_types, "1.23.0", Some(44265), None), - // Infer static outlives requirements (RFC 2093). + // Allows defining `trait X = A + B;` alias items. + (active, trait_alias, "1.24.0", Some(41517), None), + + // Allows infering `'static` outlives requirements (RFC 2093). (active, infer_static_outlives_requirements, "1.26.0", Some(54185), None), // Allows macro invocations in `extern {}` blocks. (active, macros_in_extern, "1.27.0", Some(49476), None), - // `existential type` - (active, existential_type, "1.28.0", Some(34511), None), + // Allows accessing fields of unions inside `const` functions. + (active, const_fn_union, "1.27.0", Some(51909), None), - // unstable `#[target_feature]` directives - (active, arm_target_feature, "1.27.0", Some(44839), None), - (active, aarch64_target_feature, "1.27.0", Some(44839), None), - (active, hexagon_target_feature, "1.27.0", Some(44839), None), - (active, powerpc_target_feature, "1.27.0", Some(44839), None), - (active, mips_target_feature, "1.27.0", Some(44839), None), - (active, avx512_target_feature, "1.27.0", Some(44839), None), - (active, mmx_target_feature, "1.27.0", Some(44839), None), - (active, sse4a_target_feature, "1.27.0", Some(44839), None), - (active, tbm_target_feature, "1.27.0", Some(44839), None), - (active, wasm_target_feature, "1.30.0", Some(44839), None), - (active, adx_target_feature, "1.32.0", Some(44839), None), - (active, cmpxchg16b_target_feature, "1.32.0", Some(44839), None), - (active, movbe_target_feature, "1.34.0", Some(44839), None), + // Allows casting raw pointers to `usize` during const eval. + (active, const_raw_ptr_to_usize_cast, "1.27.0", Some(51910), None), - // Allows macro invocations on modules expressions and statements and - // procedural macros to expand to non-items. - (active, proc_macro_hygiene, "1.30.0", Some(54727), None), + // Allows dereferencing raw pointers during const eval. + (active, const_raw_ptr_deref, "1.27.0", Some(51911), None), - // `#[doc(alias = "...")]` + // Allows comparing raw pointers during const eval. + (active, const_compare_raw_pointers, "1.27.0", Some(53020), None), + + // Allows `#[doc(alias = "...")]`. (active, doc_alias, "1.27.0", Some(50146), None), - // inconsistent bounds in where clauses + // Allows inconsistent bounds in where clauses. (active, trivial_bounds, "1.28.0", Some(48214), None), - // `'a: { break 'a; }` + // Allows `'a: { break 'a; }`. (active, label_break_value, "1.28.0", Some(48594), None), - // Exhaustive pattern matching on `usize` and `isize`. - (active, precise_pointer_size_matching, "1.32.0", Some(56354), None), - - // `#[doc(keyword = "...")]` + // Allows using `#[doc(keyword = "...")]`. (active, doc_keyword, "1.28.0", Some(51315), None), // Allows async and await syntax. (active, async_await, "1.28.0", Some(50547), None), - // `#[alloc_error_handler]` + // Allows reinterpretation of the bits of a value of one type as another type during const eval. + (active, const_transmute, "1.29.0", Some(53605), None), + + // Allows using `try {...}` expressions. + (active, try_blocks, "1.29.0", Some(31436), None), + + // Allows defining an `#[alloc_error_handler]`. (active, alloc_error_handler, "1.29.0", Some(51540), None), + // Allows using the `amdgpu-kernel` ABI. (active, abi_amdgpu_kernel, "1.29.0", Some(51575), None), - // Added for testing E0705; perma-unstable. - (active, test_2018_feature, "1.31.0", Some(0), Some(Edition::Edition2018)), + // Allows panicking during const eval (producing compile-time errors). + (active, const_panic, "1.30.0", Some(51999), None), + + // Allows `#[marker]` on certain traits allowing overlapping implementations. + (active, marker_trait_attr, "1.30.0", Some(29864), None), + + // Allows macro invocations on modules expressions and statements and + // procedural macros to expand to non-items. + (active, proc_macro_hygiene, "1.30.0", Some(54727), None), // Allows unsized rvalues at arguments and parameters. (active, unsized_locals, "1.30.0", Some(48055), None), - // `#![test_runner]` - // `#[test_case]` + // Allows custom test frameworks with `#![test_runner]` and `#[test_case]`. (active, custom_test_frameworks, "1.30.0", Some(50297), None), - // non-builtin attributes in inner attribute position + // Allows non-builtin attributes in inner attribute position. (active, custom_inner_attributes, "1.30.0", Some(54726), None), - // Allow mixing of bind-by-move in patterns and references to - // those identifiers in guards, *if* we are using MIR-borrowck - // (aka NLL). Essentially this means you need to be using the - // 2018 edition or later. + // Allows mixing bind-by-move in patterns and references to those identifiers in guards. (active, bind_by_move_pattern_guards, "1.30.0", Some(15287), None), // Allows `impl Trait` in bindings (`let`, `const`, `static`). - (active, impl_trait_in_bindings, "1.30.0", Some(34511), None), + (active, impl_trait_in_bindings, "1.30.0", Some(63065), None), - // Allows `const _: TYPE = VALUE`. - (active, underscore_const_names, "1.31.0", Some(54912), None), - - // Adds `reason` and `expect` lint attributes. + // Allows using `reason` in lint attributes and the `#[expect(lint)]` lint check. (active, lint_reasons, "1.31.0", Some(54503), None), - // Allows paths to enum variants on type aliases. - (active, type_alias_enum_variants, "1.31.0", Some(49683), None), + // Allows exhaustive integer pattern matching on `usize` and `isize`. + (active, precise_pointer_size_matching, "1.32.0", Some(56354), None), - // Re-Rebalance coherence + // Allows relaxing the coherence rules such that + // `impl<T> ForeignTrait<LocalType> for ForeignType<T> is permitted. (active, re_rebalance_coherence, "1.32.0", Some(55437), None), - // Const generic types. + // Allows using `#[ffi_returns_twice]` on foreign functions. + (active, ffi_returns_twice, "1.34.0", Some(58314), None), + + // Allows const generic types (e.g. `struct Foo<const N: usize>(...);`). (active, const_generics, "1.34.0", Some(44580), None), - // #[optimize(X)] + // Allows using `#[optimize(X)]`. (active, optimize_attribute, "1.34.0", Some(54882), None), - // #[repr(align(X))] on enums - (active, repr_align_enum, "1.34.0", Some(57996), None), - - // Allows the use of C-variadics + // Allows using C-variadics. (active, c_variadic, "1.34.0", Some(44930), None), + + // Allows the user of associated type bounds. + (active, associated_type_bounds, "1.34.0", Some(52662), None), + + // Attributes on formal function params. + (active, param_attrs, "1.36.0", Some(60406), None), + + // Allows calling constructor functions in `const fn`. + (active, const_constructor, "1.37.0", Some(61456), None), + + // Allows `if/while p && let q = r && ...` chains. + (active, let_chains, "1.37.0", Some(53667), None), + + // Allows #[repr(transparent)] on enums (RFC 2645). + (active, transparent_enums, "1.37.0", Some(60405), None), + + // Allows #[repr(transparent)] on unions (RFC 2645). + (active, transparent_unions, "1.37.0", Some(60405), None), + + // Allows explicit discriminants on non-unit enum variants. + (active, arbitrary_enum_discriminant, "1.37.0", Some(60553), None), + + // Allows `impl Trait` with multiple unrelated lifetimes. + (active, member_constraints, "1.37.0", Some(61977), None), + + // Allows `async || body` closures. + (active, async_closure, "1.37.0", Some(62290), None), + + // Allows the use of `#[cfg(doctest)]`, set when rustdoc is collecting doctests + (active, cfg_doctest, "1.37.0", Some(62210), None), + + // Allows `[x; N]` where `x` is a constant (RFC 2203). + (active, const_in_array_repeat_expressions, "1.37.0", Some(49147), None), + + // Allows `impl Trait` to be used inside type aliases (RFC 2515). + (active, type_alias_impl_trait, "1.38.0", Some(63063), None), + + // ------------------------------------------------------------------------- + // feature-group-end: actual feature gates + // ------------------------------------------------------------------------- ); +/// Some features are known to be incomplete and using them is likely to have +/// unanticipated results, such as compiler crashes. We warn the user about these +/// to alert them. +pub const INCOMPLETE_FEATURES: &[Symbol] = &[ + sym::impl_trait_in_bindings, + sym::generic_associated_types, + sym::const_generics, + sym::let_chains, +]; + declare_features! ( + // ------------------------------------------------------------------------- + // feature-group-start: removed features + // ------------------------------------------------------------------------- + (removed, import_shadowing, "1.0.0", None, None, None), (removed, managed_boxes, "1.0.0", None, None, None), // Allows use of unary negate on unsigned integers, e.g., -e for e: u8 @@ -490,16 +593,17 @@ declare_features! ( (removed, visible_private_types, "1.0.0", None, None, None), (removed, unsafe_no_drop_flag, "1.0.0", None, None, None), // Allows using items which are missing stability attributes - // rustc internal (removed, unmarked_api, "1.0.0", None, None, None), - (removed, pushpop_unsafe, "1.2.0", None, None, None), (removed, allocator, "1.0.0", None, None, None), (removed, simd, "1.0.0", Some(27731), None, Some("removed in favor of `#[repr(simd)]`")), - (removed, advanced_slice_patterns, "1.0.0", Some(23121), None, + (removed, advanced_slice_patterns, "1.0.0", Some(62254), None, Some("merged into `#![feature(slice_patterns)]`")), (removed, macro_reexport, "1.0.0", Some(29638), None, Some("subsumed by `pub use`")), + (removed, pushpop_unsafe, "1.2.0", None, None, None), + (removed, needs_allocator, "1.4.0", Some(27389), None, + Some("subsumed by `#![feature(allocator_internals)]`")), (removed, proc_macro_mod, "1.27.0", Some(54727), None, Some("subsumed by `#![feature(proc_macro_hygiene)]`")), (removed, proc_macro_expr, "1.27.0", Some(54727), None, @@ -511,12 +615,23 @@ declare_features! ( (removed, panic_implementation, "1.28.0", Some(44489), None, Some("subsumed by `#[panic_handler]`")), // Allows the use of `#[derive(Anything)]` as sugar for `#[derive_Anything]`. - (removed, custom_derive, "1.0.0", Some(29644), None, + (removed, custom_derive, "1.32.0", Some(29644), None, Some("subsumed by `#[proc_macro_derive]`")), // Paths of the form: `extern::foo::bar` (removed, extern_in_paths, "1.33.0", Some(55600), None, Some("subsumed by `::foo::bar` paths")), - (removed, quote, "1.0.0", Some(29601), None, None), + (removed, quote, "1.33.0", Some(29601), None, None), + // Allows using `#[unsafe_destructor_blind_to_params]` (RFC 1238). + (removed, dropck_parametricity, "1.38.0", Some(28498), None, None), + (removed, await_macro, "1.38.0", Some(50547), None, + Some("subsumed by `.await` syntax")), + // Allows defining `existential type`s. + (removed, existential_type, "1.38.0", Some(63063), None, + Some("removed in favor of `#![feature(type_alias_impl_trait)]`")), + + // ------------------------------------------------------------------------- + // feature-group-end: removed features + // ------------------------------------------------------------------------- ); declare_features! ( @@ -524,63 +639,85 @@ declare_features! ( ); declare_features! ( - (accepted, associated_types, "1.0.0", None, None), - // Allows overloading augmented assignment operations like `a += b`. - (accepted, augmented_assignments, "1.8.0", Some(28235), None), - // Allows empty structs and enum variants with braces. - (accepted, braced_empty_structs, "1.8.0", Some(29720), None), - // Allows indexing into constant arrays. - (accepted, const_indexing, "1.26.0", Some(29947), None), - (accepted, default_type_params, "1.0.0", None, None), - (accepted, globs, "1.0.0", None, None), - (accepted, if_let, "1.0.0", None, None), + // ------------------------------------------------------------------------- + // feature-group-start: for testing purposes + // ------------------------------------------------------------------------- + // A temporary feature gate used to enable parser extensions needed // to bootstrap fix for #5723. (accepted, issue_5723_bootstrap, "1.0.0", None, None), + // These are used to test this portion of the compiler, + // they don't actually mean anything. + (accepted, test_accepted_feature, "1.0.0", None, None), + + // ------------------------------------------------------------------------- + // feature-group-end: for testing purposes + // ------------------------------------------------------------------------- + + // ------------------------------------------------------------------------- + // feature-group-start: accepted features + // ------------------------------------------------------------------------- + + // Allows using associated `type`s in `trait`s. + (accepted, associated_types, "1.0.0", None, None), + // Allows using assigning a default type to type parameters in algebraic data type definitions. + (accepted, default_type_params, "1.0.0", None, None), + // FIXME: explain `globs`. + (accepted, globs, "1.0.0", None, None), + // Allows `macro_rules!` items. (accepted, macro_rules, "1.0.0", None, None), - // Allows using `#![no_std]`. - (accepted, no_std, "1.6.0", None, None), + // Allows use of `&foo[a..b]` as a slicing syntax. (accepted, slicing_syntax, "1.0.0", None, None), + // Allows struct variants `Foo { baz: u8, .. }` in enums (RFC 418). (accepted, struct_variant, "1.0.0", None, None), - // These are used to test this portion of the compiler, they don't actually - // mean anything. - (accepted, test_accepted_feature, "1.0.0", None, None), + // Allows indexing tuples. (accepted, tuple_indexing, "1.0.0", None, None), - // Allows macros to appear in the type position. - (accepted, type_macros, "1.13.0", Some(27245), None), + // Allows the use of `if let` expressions. + (accepted, if_let, "1.0.0", None, None), + // Allows the use of `while let` expressions. (accepted, while_let, "1.0.0", None, None), + // Allows using `#![no_std]`. + (accepted, no_std, "1.6.0", None, None), + // Allows overloading augmented assignment operations like `a += b`. + (accepted, augmented_assignments, "1.8.0", Some(28235), None), + // Allows empty structs and enum variants with braces. + (accepted, braced_empty_structs, "1.8.0", Some(29720), None), // Allows `#[deprecated]` attribute. (accepted, deprecated, "1.9.0", Some(29935), None), - // `expr?` + // Allows macros to appear in the type position. + (accepted, type_macros, "1.13.0", Some(27245), None), + // Allows use of the postfix `?` operator in expressions. (accepted, question_mark, "1.13.0", Some(31436), None), // Allows `..` in tuple (struct) patterns. (accepted, dotdot_in_tuple_patterns, "1.14.0", Some(33627), None), + // Allows some increased flexibility in the name resolution rules, + // especially around globs and shadowing (RFC 1560). (accepted, item_like_imports, "1.15.0", Some(35120), None), // Allows using `Self` and associated types in struct expressions and patterns. (accepted, more_struct_aliases, "1.16.0", Some(37544), None), - // elide `'static` lifetimes in `static`s and `const`s. + // Allows elision of `'static` lifetimes in `static`s and `const`s. (accepted, static_in_const, "1.17.0", Some(35897), None), // Allows field shorthands (`x` meaning `x: x`) in struct literal expressions. (accepted, field_init_shorthand, "1.17.0", Some(37340), None), // Allows the definition recursive static items. (accepted, static_recursion, "1.17.0", Some(29719), None), - // `pub(restricted)` visibilities (RFC 1422) + // Allows `pub(restricted)` visibilities (RFC 1422). (accepted, pub_restricted, "1.18.0", Some(32409), None), - // `#![windows_subsystem]` + // Allows `#![windows_subsystem]`. (accepted, windows_subsystem, "1.18.0", Some(37499), None), // Allows `break {expr}` with a value inside `loop`s. (accepted, loop_break_value, "1.19.0", Some(37339), None), - // Permits numeric fields in struct expressions and patterns. + // Allows numeric fields in struct expressions and patterns. (accepted, relaxed_adts, "1.19.0", Some(35626), None), - // Coerces non capturing closures to function pointers. + // Allows coercing non capturing closures to function pointers. (accepted, closure_to_fn_coercion, "1.19.0", Some(39817), None), // Allows attributes on struct literal fields. (accepted, struct_field_attributes, "1.20.0", Some(38814), None), // Allows the definition of associated constants in `trait` or `impl` blocks. (accepted, associated_consts, "1.20.0", Some(29646), None), - // Usage of the `compile_error!` macro. + // Allows usage of the `compile_error!` macro. (accepted, compile_error, "1.20.0", Some(40872), None), - // See rust-lang/rfcs#1414. Allows code like `let x: &'static u32 = &42` to work. + // Allows code like `let x: &'static u32 = &42` to work (RFC 1414). (accepted, rvalue_static_promotion, "1.21.0", Some(38865), None), // Allows `Drop` types in constants (RFC 1440). (accepted, drop_types_in_const, "1.22.0", Some(33156), None), @@ -591,24 +728,27 @@ declare_features! ( (accepted, repr_align, "1.25.0", Some(33626), None), // Allows '|' at beginning of match arms (RFC 1925). (accepted, match_beginning_vert, "1.25.0", Some(44101), None), - // Nested groups in `use` (RFC 2128) + // Allows nested groups in `use` items (RFC 2128). (accepted, use_nested_groups, "1.25.0", Some(44494), None), - // `a..=b` and `..=b` + // Allows indexing into constant arrays. + (accepted, const_indexing, "1.26.0", Some(29947), None), + // Allows using `a..=b` and `..=b` as inclusive range syntaxes. (accepted, inclusive_range_syntax, "1.26.0", Some(28237), None), // Allows `..=` in patterns (RFC 1192). (accepted, dotdoteq_in_patterns, "1.26.0", Some(28237), None), - // Termination trait in main (RFC 1937) + // Allows `fn main()` with return types which implements `Termination` (RFC 1937). (accepted, termination_trait, "1.26.0", Some(43301), None), - // `Copy`/`Clone` closures (RFC 2132). + // Allows implementing `Clone` for closures where possible (RFC 2132). (accepted, clone_closures, "1.26.0", Some(44490), None), + // Allows implementing `Copy` for closures where possible (RFC 2132). (accepted, copy_closures, "1.26.0", Some(44490), None), // Allows `impl Trait` in function arguments. (accepted, universal_impl_trait, "1.26.0", Some(34511), None), // Allows `impl Trait` in function return types. (accepted, conservative_impl_trait, "1.26.0", Some(34511), None), - // The `i128` type + // Allows using the `u128` and `i128` types. (accepted, i128_type, "1.26.0", Some(35118), None), - // Default match binding modes (RFC 2005) + // Allows default match binding modes (RFC 2005). (accepted, match_default_bindings, "1.26.0", Some(42640), None), // Allows `'_` placeholder lifetimes. (accepted, underscore_lifetimes, "1.26.0", Some(44524), None), @@ -618,21 +758,21 @@ declare_features! ( (accepted, cfg_target_feature, "1.27.0", Some(29717), None), // Allows `#[target_feature(...)]`. (accepted, target_feature, "1.27.0", None, None), - // Trait object syntax with `dyn` prefix + // Allows using `dyn Trait` as a syntax for trait objects. (accepted, dyn_trait, "1.27.0", Some(44662), None), // Allows `#[must_use]` on functions, and introduces must-use operators (RFC 1940). (accepted, fn_must_use, "1.27.0", Some(43302), None), // Allows use of the `:lifetime` macro fragment specifier. (accepted, macro_lifetime_matcher, "1.27.0", Some(34303), None), - // Termination trait in tests (RFC 1937) + // Allows `#[test]` functions where the return type implements `Termination` (RFC 1937). (accepted, termination_trait_test, "1.27.0", Some(48854), None), - // The `#[global_allocator]` attribute + // Allows the `#[global_allocator]` attribute. (accepted, global_allocator, "1.28.0", Some(27389), None), // Allows `#[repr(transparent)]` attribute on newtype structs. (accepted, repr_transparent, "1.28.0", Some(43036), None), - // Procedural macros in `proc-macro` crates + // Allows procedural macros in `proc-macro` crates. (accepted, proc_macro, "1.29.0", Some(38356), None), - // `foo.rs` as an alternative to `foo/mod.rs` + // Allows `foo.rs` as an alternative to `foo/mod.rs`. (accepted, non_modrs_mods, "1.30.0", Some(44660), None), // Allows use of the `:vis` macro fragment specifier (accepted, macro_vis_matcher, "1.30.0", Some(41022), None), @@ -641,47 +781,50 @@ declare_features! ( (accepted, use_extern_macros, "1.30.0", Some(35896), None), // Allows keywords to be escaped for use as identifiers. (accepted, raw_identifiers, "1.30.0", Some(48589), None), - // Attributes scoped to tools. + // Allows attributes scoped to tools. (accepted, tool_attributes, "1.30.0", Some(44690), None), // Allows multi-segment paths in attributes and derives. (accepted, proc_macro_path_invoc, "1.30.0", Some(38356), None), // Allows all literals in attribute lists and values of key-value pairs. (accepted, attr_literals, "1.30.0", Some(34981), None), - // Infer outlives requirements (RFC 2093). + // Allows inferring outlives requirements (RFC 2093). (accepted, infer_outlives_requirements, "1.30.0", Some(44493), None), + // Allows annotating functions conforming to `fn(&PanicInfo) -> !` with `#[panic_handler]`. + // This defines the behavior of panics. (accepted, panic_handler, "1.30.0", Some(44489), None), - // Used to preserve symbols (see llvm.used). + // Allows `#[used]` to preserve symbols (see llvm.used). (accepted, used, "1.30.0", Some(40289), None), - // `crate` in paths + // Allows `crate` in paths. (accepted, crate_in_paths, "1.30.0", Some(45477), None), - // Resolve absolute paths as paths from other crates. + // Allows resolving absolute paths as paths from other crates. (accepted, extern_absolute_paths, "1.30.0", Some(44660), None), - // Access to crate names passed via `--extern` through prelude. + // Allows access to crate names passed via `--extern` through prelude. (accepted, extern_prelude, "1.30.0", Some(44660), None), - // Parentheses in patterns + // Allows parentheses in patterns. (accepted, pattern_parentheses, "1.31.0", Some(51087), None), // Allows the definition of `const fn` functions. (accepted, min_const_fn, "1.31.0", Some(53555), None), - // Scoped lints + // Allows scoped lints. (accepted, tool_lints, "1.31.0", Some(44690), None), - // `impl<I:Iterator> Iterator for &mut Iterator` - // `impl Debug for Foo<'_>` + // Allows lifetime elision in `impl` headers. For example: + // + `impl<I:Iterator> Iterator for &mut Iterator` + // + `impl Debug for Foo<'_>` (accepted, impl_header_lifetime_elision, "1.31.0", Some(15872), None), - // `extern crate foo as bar;` puts `bar` into extern prelude. + // Allows `extern crate foo as bar;`. This puts `bar` into extern prelude. (accepted, extern_crate_item_prelude, "1.31.0", Some(55599), None), // Allows use of the `:literal` macro fragment specifier (RFC 1576). (accepted, macro_literal_matcher, "1.32.0", Some(35625), None), - // Use `?` as the Kleene "at most one" operator. + // Allows use of `?` as the Kleene "at most one" operator in macros. (accepted, macro_at_most_once_rep, "1.32.0", Some(48075), None), - // `Self` struct constructor (RFC 2302) + // Allows `Self` struct constructor (RFC 2302). (accepted, self_struct_ctor, "1.32.0", Some(51994), None), - // `Self` in type definitions (RFC 2300) + // Allows `Self` in type definitions (RFC 2300). (accepted, self_in_typedefs, "1.32.0", Some(49303), None), // Allows `use x::y;` to search `x` in the current scope. (accepted, uniform_paths, "1.32.0", Some(53130), None), - // Integer match exhaustiveness checking (RFC 2591) + // Allows integer match exhaustiveness checking (RFC 2591). (accepted, exhaustive_integer_patterns, "1.33.0", Some(50907), None), - // `use path as _;` and `extern crate c as _;` + // Allows `use path as _;` and `extern crate c as _;`. (accepted, underscore_imports, "1.33.0", Some(48216), None), // Allows `#[repr(packed(N))]` attribute on structs. (accepted, repr_packed, "1.33.0", Some(33158), None), @@ -693,16 +836,28 @@ declare_features! ( // As long as control flow is not implemented in const eval, `&&` and `||` may not be used // at the same time as let bindings. (accepted, const_let, "1.33.0", Some(48821), None), - // `#[cfg_attr(predicate, multiple, attributes, here)]` + // Allows `#[cfg_attr(predicate, multiple, attributes, here)]`. (accepted, cfg_attr_multi, "1.33.0", Some(54881), None), - // Top level or-patterns (`p | q`) in `if let` and `while let`. + // Allows top level or-patterns (`p | q`) in `if let` and `while let`. (accepted, if_while_or_patterns, "1.33.0", Some(48215), None), // Allows `cfg(target_vendor = "...")`. (accepted, cfg_target_vendor, "1.33.0", Some(29718), None), - // `extern crate self as foo;` puts local crate root into extern prelude under name `foo`. + // Allows `extern crate self as foo;`. + // This puts local crate root into extern prelude under name `foo`. (accepted, extern_crate_self, "1.34.0", Some(56409), None), - // support for arbitrary delimited token streams in non-macro attributes + // Allows arbitrary delimited token streams in non-macro attributes. (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208), None), + // Allows paths to enum variants on type aliases including `Self`. + (accepted, type_alias_enum_variants, "1.37.0", Some(49683), None), + // Allows using `#[repr(align(X))]` on enums with equivalent semantics + // to wrapping an enum in a wrapper struct with `#[repr(align(X))]`. + (accepted, repr_align_enum, "1.37.0", Some(57996), None), + // Allows `const _: TYPE = VALUE`. + (accepted, underscore_const_names, "1.37.0", Some(54912), None), + + // ------------------------------------------------------------------------- + // feature-group-end: accepted features + // ------------------------------------------------------------------------- ); // If you change this, please modify `src/doc/unstable-book` as well. You must @@ -727,33 +882,12 @@ pub enum AttributeType { pub enum AttributeGate { /// Is gated by a given feature gate, reason /// and function to check if enabled - Gated(Stability, &'static str, &'static str, fn(&Features) -> bool), + Gated(Stability, Symbol, &'static str, fn(&Features) -> bool), /// Ungated attribute, can be used on all release channels Ungated, } -/// A template that the attribute input must match. -/// Only top-level shape (`#[attr]` vs `#[attr(...)]` vs `#[attr = ...]`) is considered now. -#[derive(Clone, Copy)] -pub struct AttributeTemplate { - word: bool, - list: Option<&'static str>, - name_value_str: Option<&'static str>, -} - -impl AttributeTemplate { - /// Checks that the given meta-item is compatible with this template. - fn compatible(&self, meta_item_kind: &ast::MetaItemKind) -> bool { - match meta_item_kind { - ast::MetaItemKind::Word => self.word, - ast::MetaItemKind::List(..) => self.list.is_some(), - ast::MetaItemKind::NameValue(lit) if lit.node.is_str() => self.name_value_str.is_some(), - ast::MetaItemKind::NameValue(..) => false, - } - } -} - /// A convenience macro for constructing attribute templates. /// E.g., `template!(Word, List: "description")` means that the attribute /// supports forms `#[attr]` and `#[attr(description)]`. @@ -811,474 +945,608 @@ macro_rules! cfg_fn { }} } -pub fn deprecated_attributes() -> Vec<&'static (&'static str, AttributeType, +pub fn deprecated_attributes() -> Vec<&'static (Symbol, AttributeType, AttributeTemplate, AttributeGate)> { BUILTIN_ATTRIBUTES.iter().filter(|(.., gate)| gate.is_deprecated()).collect() } pub fn is_builtin_attr_name(name: ast::Name) -> bool { - BUILTIN_ATTRIBUTES.iter().any(|&(builtin_name, ..)| name == builtin_name) + BUILTIN_ATTRIBUTE_MAP.get(&name).is_some() } pub fn is_builtin_attr(attr: &ast::Attribute) -> bool { - BUILTIN_ATTRIBUTES.iter().any(|&(builtin_name, ..)| attr.path == builtin_name) + attr.ident().and_then(|ident| BUILTIN_ATTRIBUTE_MAP.get(&ident.name)).is_some() } -// Attributes that have a special meaning to rustc or rustdoc -pub const BUILTIN_ATTRIBUTES: &[(&str, AttributeType, AttributeTemplate, AttributeGate)] = &[ +/// Attributes that have a special meaning to rustc or rustdoc +pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ // Normal attributes - ("warn", Normal, template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), Ungated), - ("allow", Normal, template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), Ungated), - ("forbid", Normal, template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), Ungated), - ("deny", Normal, template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), Ungated), - - ("macro_use", Normal, template!(Word, List: "name1, name2, ..."), Ungated), - ("macro_export", Normal, template!(Word, List: "local_inner_macros"), Ungated), - ("plugin_registrar", Normal, template!(Word), Ungated), - - ("cfg", Normal, template!(List: "predicate"), Ungated), - ("cfg_attr", Normal, template!(List: "predicate, attr1, attr2, ..."), Ungated), - ("main", Normal, template!(Word), Ungated), - ("start", Normal, template!(Word), Ungated), - ("repr", Normal, template!(List: "C, packed, ..."), Ungated), - ("path", Normal, template!(NameValueStr: "file"), Ungated), - ("automatically_derived", Normal, template!(Word), Ungated), - ("no_mangle", Normal, template!(Word), Ungated), - ("no_link", Normal, template!(Word), Ungated), - ("derive", Normal, template!(List: "Trait1, Trait2, ..."), Ungated), - ("should_panic", Normal, template!(Word, List: r#"expected = "reason"#, NameValueStr: "reason"), - Ungated), - ("ignore", Normal, template!(Word, NameValueStr: "reason"), Ungated), - ("no_implicit_prelude", Normal, template!(Word), Ungated), - ("reexport_test_harness_main", Normal, template!(NameValueStr: "name"), Ungated), - ("link_args", Normal, template!(NameValueStr: "args"), Gated(Stability::Unstable, - "link_args", + ( + sym::warn, + Normal, + template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), + Ungated + ), + ( + sym::allow, + Normal, + template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), + Ungated + ), + ( + sym::forbid, + Normal, + template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), + Ungated + ), + ( + sym::deny, + Normal, + template!(List: r#"lint1, lint2, ..., /*opt*/ reason = "...""#), + Ungated + ), + + (sym::macro_use, Normal, template!(Word, List: "name1, name2, ..."), Ungated), + (sym::macro_export, Normal, template!(Word, List: "local_inner_macros"), Ungated), + (sym::plugin_registrar, Normal, template!(Word), Ungated), + + (sym::cfg, Normal, template!(List: "predicate"), Ungated), + (sym::cfg_attr, Normal, template!(List: "predicate, attr1, attr2, ..."), Ungated), + (sym::main, Normal, template!(Word), Ungated), + (sym::start, Normal, template!(Word), Ungated), + (sym::repr, Normal, template!(List: "C, packed, ..."), Ungated), + (sym::path, Normal, template!(NameValueStr: "file"), Ungated), + (sym::automatically_derived, Normal, template!(Word), Ungated), + (sym::no_mangle, Whitelisted, template!(Word), Ungated), + (sym::no_link, Normal, template!(Word), Ungated), + (sym::derive, Normal, template!(List: "Trait1, Trait2, ..."), Ungated), + ( + sym::should_panic, + Normal, + template!(Word, List: r#"expected = "reason"#, NameValueStr: "reason"), + Ungated + ), + (sym::ignore, Normal, template!(Word, NameValueStr: "reason"), Ungated), + (sym::no_implicit_prelude, Normal, template!(Word), Ungated), + (sym::reexport_test_harness_main, Normal, template!(NameValueStr: "name"), Ungated), + (sym::link_args, Normal, template!(NameValueStr: "args"), Gated(Stability::Unstable, + sym::link_args, "the `link_args` attribute is experimental and not \ - portable across platforms, it is recommended to \ - use `#[link(name = \"foo\")] instead", + portable across platforms, it is recommended to \ + use `#[link(name = \"foo\")] instead", cfg_fn!(link_args))), - ("macro_escape", Normal, template!(Word), Ungated), + (sym::macro_escape, Normal, template!(Word), Ungated), // RFC #1445. - ("structural_match", Whitelisted, template!(Word), Gated(Stability::Unstable, - "structural_match", + (sym::structural_match, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::structural_match, "the semantics of constant patterns is \ - not yet settled", + not yet settled", cfg_fn!(structural_match))), // RFC #2008 - ("non_exhaustive", Whitelisted, template!(Word), Gated(Stability::Unstable, - "non_exhaustive", - "non exhaustive is an experimental feature", - cfg_fn!(non_exhaustive))), + (sym::non_exhaustive, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::non_exhaustive, + "non exhaustive is an experimental feature", + cfg_fn!(non_exhaustive))), // RFC #1268 - ("marker", Normal, template!(Word), Gated(Stability::Unstable, - "marker_trait_attr", - "marker traits is an experimental feature", - cfg_fn!(marker_trait_attr))), - - ("plugin", CrateLevel, template!(List: "name|name(args)"), Gated(Stability::Unstable, - "plugin", - "compiler plugins are experimental \ - and possibly buggy", - cfg_fn!(plugin))), - - ("no_std", CrateLevel, template!(Word), Ungated), - ("no_core", CrateLevel, template!(Word), Gated(Stability::Unstable, - "no_core", - "no_core is experimental", - cfg_fn!(no_core))), - ("lang", Normal, template!(NameValueStr: "name"), Gated(Stability::Unstable, - "lang_items", - "language items are subject to change", - cfg_fn!(lang_items))), - ("linkage", Whitelisted, template!(NameValueStr: "external|internal|..."), - Gated(Stability::Unstable, - "linkage", - "the `linkage` attribute is experimental \ + (sym::marker, Normal, template!(Word), Gated(Stability::Unstable, + sym::marker_trait_attr, + "marker traits is an experimental feature", + cfg_fn!(marker_trait_attr))), + + (sym::plugin, CrateLevel, template!(List: "name|name(args)"), Gated(Stability::Unstable, + sym::plugin, + "compiler plugins are experimental \ + and possibly buggy", + cfg_fn!(plugin))), + + (sym::no_std, CrateLevel, template!(Word), Ungated), + (sym::no_core, CrateLevel, template!(Word), Gated(Stability::Unstable, + sym::no_core, + "no_core is experimental", + cfg_fn!(no_core))), + (sym::lang, Normal, template!(NameValueStr: "name"), Gated(Stability::Unstable, + sym::lang_items, + "language items are subject to change", + cfg_fn!(lang_items))), + (sym::linkage, Whitelisted, template!(NameValueStr: "external|internal|..."), + Gated(Stability::Unstable, + sym::linkage, + "the `linkage` attribute is experimental \ and not portable across platforms", - cfg_fn!(linkage))), - ("thread_local", Whitelisted, template!(Word), Gated(Stability::Unstable, - "thread_local", + cfg_fn!(linkage))), + (sym::thread_local, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::thread_local, "`#[thread_local]` is an experimental feature, and does \ - not currently handle destructors.", + not currently handle destructors", cfg_fn!(thread_local))), - ("rustc_on_unimplemented", Normal, template!(List: - r#"/*opt*/ message = "...", /*opt*/ label = "...", /*opt*/ note = "...""#, - NameValueStr: "message"), - Gated(Stability::Unstable, - "on_unimplemented", - "the `#[rustc_on_unimplemented]` attribute \ - is an experimental feature", - cfg_fn!(on_unimplemented))), - ("rustc_const_unstable", Normal, template!(List: r#"feature = "name""#), - Gated(Stability::Unstable, - "rustc_const_unstable", - "the `#[rustc_const_unstable]` attribute \ - is an internal feature", - cfg_fn!(rustc_const_unstable))), - ("global_allocator", Normal, template!(Word), Ungated), - ("default_lib_allocator", Whitelisted, template!(Word), Gated(Stability::Unstable, - "allocator_internals", + (sym::rustc_on_unimplemented, Whitelisted, template!(List: + r#"/*opt*/ message = "...", /*opt*/ label = "...", /*opt*/ note = "...""#, + NameValueStr: "message"), + Gated(Stability::Unstable, + sym::on_unimplemented, + "the `#[rustc_on_unimplemented]` attribute \ + is an experimental feature", + cfg_fn!(on_unimplemented))), + (sym::rustc_const_unstable, Normal, template!(List: r#"feature = "name""#), + Gated(Stability::Unstable, + sym::rustc_const_unstable, + "the `#[rustc_const_unstable]` attribute \ + is an internal feature", + cfg_fn!(rustc_const_unstable))), + (sym::default_lib_allocator, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::allocator_internals, "the `#[default_lib_allocator]` \ - attribute is an experimental feature", + attribute is an experimental feature", cfg_fn!(allocator_internals))), - ("needs_allocator", Normal, template!(Word), Gated(Stability::Unstable, - "allocator_internals", - "the `#[needs_allocator]` \ - attribute is an experimental \ - feature", - cfg_fn!(allocator_internals))), - ("panic_runtime", Whitelisted, template!(Word), Gated(Stability::Unstable, - "panic_runtime", - "the `#[panic_runtime]` attribute is \ - an experimental feature", - cfg_fn!(panic_runtime))), - ("needs_panic_runtime", Whitelisted, template!(Word), Gated(Stability::Unstable, - "needs_panic_runtime", - "the `#[needs_panic_runtime]` \ + (sym::needs_allocator, Normal, template!(Word), Gated(Stability::Unstable, + sym::allocator_internals, + "the `#[needs_allocator]` \ + attribute is an experimental \ + feature", + cfg_fn!(allocator_internals))), + (sym::panic_runtime, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::panic_runtime, + "the `#[panic_runtime]` attribute is \ + an experimental feature", + cfg_fn!(panic_runtime))), + (sym::needs_panic_runtime, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::needs_panic_runtime, + "the `#[needs_panic_runtime]` \ attribute is an experimental \ feature", - cfg_fn!(needs_panic_runtime))), - ("rustc_outlives", Normal, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_outlives]` attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_variance", Normal, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_variance]` attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_layout", Normal, template!(List: "field1, field2, ..."), - Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_layout]` attribute \ + cfg_fn!(needs_panic_runtime))), + (sym::rustc_outlives, Normal, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_outlives]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_variance, Normal, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_variance]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_layout, Normal, template!(List: "field1, field2, ..."), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_layout]` attribute \ is just used for rustc unit tests \ and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_regions", Normal, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", + cfg_fn!(rustc_attrs))), + (sym::rustc_layout_scalar_valid_range_start, Whitelisted, template!(List: "value"), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_layout_scalar_valid_range_start]` attribute \ + is just used to enable niche optimizations in libcore \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_layout_scalar_valid_range_end, Whitelisted, template!(List: "value"), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_layout_scalar_valid_range_end]` attribute \ + is just used to enable niche optimizations in libcore \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_nonnull_optimization_guaranteed, Whitelisted, template!(Word), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_nonnull_optimization_guaranteed]` attribute \ + is just used to enable niche optimizations in libcore \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_regions, Normal, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, "the `#[rustc_regions]` attribute \ - is just used for rustc unit tests \ - and will never be stable", + is just used for rustc unit tests \ + and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_error", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_error]` attribute \ + (sym::rustc_error, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_error]` attribute \ is just used for rustc unit tests \ and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_dump_user_substs", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "this attribute \ + cfg_fn!(rustc_attrs))), + (sym::rustc_dump_user_substs, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "this attribute \ is just used for rustc unit tests \ and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_if_this_changed", Whitelisted, template!(Word, List: "DepNode"), - Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_if_this_changed]` attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_then_this_would_need", Whitelisted, template!(List: "DepNode"), - Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_if_this_changed]` attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_dirty", Whitelisted, template!(List: r#"cfg = "...", /*opt*/ label = "...", + cfg_fn!(rustc_attrs))), + (sym::rustc_if_this_changed, Whitelisted, template!(Word, List: "DepNode"), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_if_this_changed]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_then_this_would_need, Whitelisted, template!(List: "DepNode"), + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_if_this_changed]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_dirty, Whitelisted, template!(List: r#"cfg = "...", /*opt*/ label = "...", /*opt*/ except = "...""#), - Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_dirty]` attribute \ + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_dirty]` attribute \ is just used for rustc unit tests \ and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_clean", Whitelisted, template!(List: r#"cfg = "...", /*opt*/ label = "...", + cfg_fn!(rustc_attrs))), + (sym::rustc_clean, Whitelisted, template!(List: r#"cfg = "...", /*opt*/ label = "...", /*opt*/ except = "...""#), - Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_clean]` attribute \ + Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_clean]` attribute \ is just used for rustc unit tests \ and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_partition_reused", Whitelisted, template!(List: r#"cfg = "...", module = "...""#), - Gated(Stability::Unstable, - "rustc_attrs", - "this attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_partition_codegened", Whitelisted, template!(List: r#"cfg = "...", module = "...""#), - Gated(Stability::Unstable, - "rustc_attrs", - "this attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_expected_cgu_reuse", Whitelisted, template!(List: r#"cfg = "...", module = "...", - kind = "...""#), + cfg_fn!(rustc_attrs))), + ( + sym::rustc_partition_reused, + Whitelisted, + template!(List: r#"cfg = "...", module = "...""#), + Gated( + Stability::Unstable, + sym::rustc_attrs, + "this attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs) + ) + ), + ( + sym::rustc_partition_codegened, + Whitelisted, + template!(List: r#"cfg = "...", module = "...""#), + Gated( + Stability::Unstable, + sym::rustc_attrs, + "this attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs), + ) + ), + (sym::rustc_expected_cgu_reuse, Whitelisted, template!(List: r#"cfg = "...", module = "...", + kind = "...""#), Gated(Stability::Unstable, - "rustc_attrs", + sym::rustc_attrs, "this attribute \ - is just used for rustc unit tests \ - and will never be stable", + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_synthetic, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "this attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_symbol_name, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal rustc attributes will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_def_path, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal rustc attributes will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_mir, Whitelisted, template!(List: "arg1, arg2, ..."), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_mir]` attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + ( + sym::rustc_inherit_overflow_checks, + Whitelisted, + template!(Word), + Gated( + Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_inherit_overflow_checks]` \ + attribute is just used to control \ + overflow checking behavior of several \ + libcore functions that are inlined \ + across crates and will never be stable", + cfg_fn!(rustc_attrs), + ) + ), + + (sym::rustc_dump_program_clauses, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_dump_program_clauses]` \ + attribute is just used for rustc unit \ + tests and will never be stable", + cfg_fn!(rustc_attrs))), + (sym::rustc_dump_env_program_clauses, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_dump_env_program_clauses]` \ + attribute is just used for rustc unit \ + tests and will never be stable", cfg_fn!(rustc_attrs))), - ("rustc_synthetic", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "this attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_symbol_name", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "internal rustc attributes will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_item_path", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "internal rustc attributes will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_mir", Whitelisted, template!(List: "arg1, arg2, ..."), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_mir]` attribute \ - is just used for rustc unit tests \ - and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_inherit_overflow_checks", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_inherit_overflow_checks]` \ - attribute is just used to control \ - overflow checking behavior of several \ - libcore functions that are inlined \ - across crates and will never be stable", - cfg_fn!(rustc_attrs))), - - ("rustc_dump_program_clauses", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_dump_program_clauses]` \ - attribute is just used for rustc unit \ - tests and will never be stable", - cfg_fn!(rustc_attrs))), - ("rustc_test_marker", Normal, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "the `#[rustc_test_marker]` attribute \ - is used internally to track tests", - cfg_fn!(rustc_attrs))), - ("rustc_transparent_macro", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "used internally for testing macro hygiene", + (sym::rustc_object_lifetime_default, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_object_lifetime_default]` \ + attribute is just used for rustc unit \ + tests and will never be stable", cfg_fn!(rustc_attrs))), - ("compiler_builtins", Whitelisted, template!(Word), Gated(Stability::Unstable, - "compiler_builtins", - "the `#[compiler_builtins]` attribute is used to \ - identify the `compiler_builtins` crate which \ - contains compiler-rt intrinsics and will never be \ - stable", - cfg_fn!(compiler_builtins))), - ("sanitizer_runtime", Whitelisted, template!(Word), Gated(Stability::Unstable, - "sanitizer_runtime", - "the `#[sanitizer_runtime]` attribute is used to \ - identify crates that contain the runtime of a \ - sanitizer and will never be stable", - cfg_fn!(sanitizer_runtime))), - ("profiler_runtime", Whitelisted, template!(Word), Gated(Stability::Unstable, - "profiler_runtime", - "the `#[profiler_runtime]` attribute is used to \ - identify the `profiler_builtins` crate which \ - contains the profiler runtime and will never be \ - stable", - cfg_fn!(profiler_runtime))), - - ("allow_internal_unstable", Normal, template!(Word, List: "feat1, feat2, ..."), - Gated(Stability::Unstable, - "allow_internal_unstable", - EXPLAIN_ALLOW_INTERNAL_UNSTABLE, - cfg_fn!(allow_internal_unstable))), - - ("allow_internal_unsafe", Normal, template!(Word), Gated(Stability::Unstable, - "allow_internal_unsafe", + (sym::rustc_test_marker, Normal, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "the `#[rustc_test_marker]` attribute \ + is used internally to track tests", + cfg_fn!(rustc_attrs))), + (sym::rustc_macro_transparency, Whitelisted, template!(NameValueStr: + "transparent|semitransparent|opaque"), + Gated(Stability::Unstable, + sym::rustc_attrs, + "used internally for testing macro hygiene", + cfg_fn!(rustc_attrs))), + (sym::compiler_builtins, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::compiler_builtins, + "the `#[compiler_builtins]` attribute is used to \ + identify the `compiler_builtins` crate which \ + contains compiler-rt intrinsics and will never be \ + stable", + cfg_fn!(compiler_builtins))), + (sym::sanitizer_runtime, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::sanitizer_runtime, + "the `#[sanitizer_runtime]` attribute is used to \ + identify crates that contain the runtime of a \ + sanitizer and will never be stable", + cfg_fn!(sanitizer_runtime))), + (sym::profiler_runtime, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::profiler_runtime, + "the `#[profiler_runtime]` attribute is used to \ + identify the `profiler_builtins` crate which \ + contains the profiler runtime and will never be \ + stable", + cfg_fn!(profiler_runtime))), + + (sym::allow_internal_unstable, Normal, template!(Word, List: "feat1, feat2, ..."), + Gated(Stability::Unstable, + sym::allow_internal_unstable, + EXPLAIN_ALLOW_INTERNAL_UNSTABLE, + cfg_fn!(allow_internal_unstable))), + + (sym::allow_internal_unsafe, Normal, template!(Word), Gated(Stability::Unstable, + sym::allow_internal_unsafe, EXPLAIN_ALLOW_INTERNAL_UNSAFE, cfg_fn!(allow_internal_unsafe))), - ("fundamental", Whitelisted, template!(Word), Gated(Stability::Unstable, - "fundamental", - "the `#[fundamental]` attribute \ + (sym::fundamental, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::fundamental, + "the `#[fundamental]` attribute \ is an experimental feature", - cfg_fn!(fundamental))), - - ("proc_macro_derive", Normal, template!(List: "TraitName, \ - /*opt*/ attributes(name1, name2, ...)"), - Ungated), - - ("rustc_copy_clone_marker", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "internal implementation detail", - cfg_fn!(rustc_attrs))), + cfg_fn!(fundamental))), + + (sym::proc_macro_derive, Normal, template!(List: "TraitName, \ + /*opt*/ attributes(name1, name2, ...)"), + Ungated), + + (sym::rustc_allocator, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal implementation detail", + cfg_fn!(rustc_attrs))), + + (sym::rustc_allocator_nounwind, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal implementation detail", + cfg_fn!(rustc_attrs))), + + (sym::rustc_builtin_macro, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal implementation detail", + cfg_fn!(rustc_attrs))), + + (sym::rustc_promotable, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal implementation detail", + cfg_fn!(rustc_attrs))), + + (sym::rustc_allow_const_fn_ptr, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "internal implementation detail", + cfg_fn!(rustc_attrs))), + + (sym::rustc_dummy, Normal, template!(Word /* doesn't matter*/), Gated(Stability::Unstable, + sym::rustc_attrs, + "used by the test suite", + cfg_fn!(rustc_attrs))), // FIXME: #14408 whitelist docs since rustdoc looks at them - ("doc", Whitelisted, template!(List: "hidden|inline|...", NameValueStr: "string"), Ungated), + ( + sym::doc, + Whitelisted, + template!(List: "hidden|inline|...", NameValueStr: "string"), + Ungated + ), // FIXME: #14406 these are processed in codegen, which happens after the // lint pass - ("cold", Whitelisted, template!(Word), Ungated), - ("naked", Whitelisted, template!(Word), Gated(Stability::Unstable, - "naked_functions", - "the `#[naked]` attribute \ - is an experimental feature", - cfg_fn!(naked_functions))), - ("ffi_returns_twice", Whitelisted, template!(Word), Gated(Stability::Unstable, - "ffi_returns_twice", - "the `#[ffi_returns_twice]` attribute \ - is an experimental feature", - cfg_fn!(ffi_returns_twice))), - ("target_feature", Whitelisted, template!(List: r#"enable = "name""#), Ungated), - ("export_name", Whitelisted, template!(NameValueStr: "name"), Ungated), - ("inline", Whitelisted, template!(Word, List: "always|never"), Ungated), - ("link", Whitelisted, template!(List: r#"name = "...", /*opt*/ kind = "dylib|static|...", - /*opt*/ cfg = "...""#), Ungated), - ("link_name", Whitelisted, template!(NameValueStr: "name"), Ungated), - ("link_section", Whitelisted, template!(NameValueStr: "name"), Ungated), - ("no_builtins", Whitelisted, template!(Word), Ungated), - ("no_mangle", Whitelisted, template!(Word), Ungated), - ("no_debug", Whitelisted, template!(Word), Gated( + (sym::cold, Whitelisted, template!(Word), Ungated), + (sym::naked, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::naked_functions, + "the `#[naked]` attribute \ + is an experimental feature", + cfg_fn!(naked_functions))), + (sym::ffi_returns_twice, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::ffi_returns_twice, + "the `#[ffi_returns_twice]` attribute \ + is an experimental feature", + cfg_fn!(ffi_returns_twice))), + (sym::target_feature, Whitelisted, template!(List: r#"enable = "name""#), Ungated), + (sym::export_name, Whitelisted, template!(NameValueStr: "name"), Ungated), + (sym::inline, Whitelisted, template!(Word, List: "always|never"), Ungated), + (sym::link, Whitelisted, template!(List: r#"name = "...", /*opt*/ kind = "dylib|static|...", + /*opt*/ cfg = "...""#), Ungated), + (sym::link_name, Whitelisted, template!(NameValueStr: "name"), Ungated), + (sym::link_section, Whitelisted, template!(NameValueStr: "name"), Ungated), + (sym::no_builtins, Whitelisted, template!(Word), Ungated), + (sym::no_debug, Whitelisted, template!(Word), Gated( Stability::Deprecated("https://github.com/rust-lang/rust/issues/29721", None), - "no_debug", + sym::no_debug, "the `#[no_debug]` attribute was an experimental feature that has been \ - deprecated due to lack of demand", + deprecated due to lack of demand", cfg_fn!(no_debug))), - ("omit_gdb_pretty_printer_section", Whitelisted, template!(Word), Gated(Stability::Unstable, - "omit_gdb_pretty_printer_section", - "the `#[omit_gdb_pretty_printer_section]` \ - attribute is just used for the Rust test \ - suite", - cfg_fn!(omit_gdb_pretty_printer_section))), - ("unsafe_destructor_blind_to_params", - Normal, - template!(Word), - Gated(Stability::Deprecated("https://github.com/rust-lang/rust/issues/34761", - Some("replace this attribute with `#[may_dangle]`")), - "dropck_parametricity", - "unsafe_destructor_blind_to_params has been replaced by \ - may_dangle and will be removed in the future", - cfg_fn!(dropck_parametricity))), - ("may_dangle", - Normal, - template!(Word), - Gated(Stability::Unstable, - "dropck_eyepatch", - "may_dangle has unstable semantics and may be removed in the future", - cfg_fn!(dropck_eyepatch))), - ("unwind", Whitelisted, template!(List: "allowed|aborts"), Gated(Stability::Unstable, - "unwind_attributes", - "#[unwind] is experimental", - cfg_fn!(unwind_attributes))), - ("used", Whitelisted, template!(Word), Ungated), + ( + sym::omit_gdb_pretty_printer_section, + Whitelisted, + template!(Word), + Gated( + Stability::Unstable, + sym::omit_gdb_pretty_printer_section, + "the `#[omit_gdb_pretty_printer_section]` \ + attribute is just used for the Rust test \ + suite", + cfg_fn!(omit_gdb_pretty_printer_section) + ) + ), + (sym::may_dangle, + Normal, + template!(Word), + Gated(Stability::Unstable, + sym::dropck_eyepatch, + "`may_dangle` has unstable semantics and may be removed in the future", + cfg_fn!(dropck_eyepatch))), + (sym::unwind, Whitelisted, template!(List: "allowed|aborts"), Gated(Stability::Unstable, + sym::unwind_attributes, + "`#[unwind]` is experimental", + cfg_fn!(unwind_attributes))), + (sym::used, Whitelisted, template!(Word), Ungated), // used in resolve - ("prelude_import", Whitelisted, template!(Word), Gated(Stability::Unstable, - "prelude_import", - "`#[prelude_import]` is for use by rustc only", - cfg_fn!(prelude_import))), + (sym::prelude_import, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::prelude_import, + "`#[prelude_import]` is for use by rustc only", + cfg_fn!(prelude_import))), // FIXME: #14407 these are only looked at on-demand so we can't // guarantee they'll have already been checked - ("rustc_deprecated", Whitelisted, template!(List: r#"since = "version", reason = "...""#), - Ungated), - ("must_use", Whitelisted, template!(Word, NameValueStr: "reason"), Ungated), - ("stable", Whitelisted, template!(List: r#"feature = "name", since = "version""#), Ungated), - ("unstable", Whitelisted, template!(List: r#"feature = "name", reason = "...", issue = "N""#), - Ungated), - ("deprecated", + ( + sym::rustc_deprecated, + Whitelisted, + template!(List: r#"since = "version", reason = "...""#), + Ungated + ), + (sym::must_use, Whitelisted, template!(Word, NameValueStr: "reason"), Ungated), + ( + sym::stable, + Whitelisted, + template!(List: r#"feature = "name", since = "version""#), + Ungated + ), + ( + sym::unstable, + Whitelisted, + template!(List: r#"feature = "name", reason = "...", issue = "N""#), + Ungated + ), + (sym::deprecated, Normal, template!( Word, - List: r#"/*opt*/ since = "version", /*opt*/ note = "reason"#, + List: r#"/*opt*/ since = "version", /*opt*/ note = "reason""#, NameValueStr: "reason" ), Ungated ), - ("rustc_paren_sugar", Normal, template!(Word), Gated(Stability::Unstable, - "unboxed_closures", + (sym::rustc_paren_sugar, Normal, template!(Word), Gated(Stability::Unstable, + sym::unboxed_closures, "unboxed_closures are still evolving", cfg_fn!(unboxed_closures))), - ("windows_subsystem", Whitelisted, template!(NameValueStr: "windows|console"), Ungated), + (sym::windows_subsystem, Whitelisted, template!(NameValueStr: "windows|console"), Ungated), - ("proc_macro_attribute", Normal, template!(Word), Ungated), - ("proc_macro", Normal, template!(Word), Ungated), + (sym::proc_macro_attribute, Normal, template!(Word), Ungated), + (sym::proc_macro, Normal, template!(Word), Ungated), - ("rustc_proc_macro_decls", Normal, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "used internally by rustc", - cfg_fn!(rustc_attrs))), + (sym::rustc_proc_macro_decls, Normal, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "used internally by rustc", + cfg_fn!(rustc_attrs))), - ("allow_fail", Normal, template!(Word), Gated(Stability::Unstable, - "allow_fail", - "allow_fail attribute is currently unstable", - cfg_fn!(allow_fail))), + (sym::allow_fail, Normal, template!(Word), Gated(Stability::Unstable, + sym::allow_fail, + "allow_fail attribute is currently unstable", + cfg_fn!(allow_fail))), - ("rustc_std_internal_symbol", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "this is an internal attribute that will \ - never be stable", - cfg_fn!(rustc_attrs))), + (sym::rustc_std_internal_symbol, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "this is an internal attribute that will \ + never be stable", + cfg_fn!(rustc_attrs))), // whitelists "identity-like" conversion methods to suggest on type mismatch - ("rustc_conversion_suggestion", Whitelisted, template!(Word), Gated(Stability::Unstable, - "rustc_attrs", - "this is an internal attribute that will \ + (sym::rustc_conversion_suggestion, Whitelisted, template!(Word), Gated(Stability::Unstable, + sym::rustc_attrs, + "this is an internal attribute that will \ never be stable", - cfg_fn!(rustc_attrs))), + cfg_fn!(rustc_attrs))), - ("rustc_args_required_const", Whitelisted, template!(List: "N"), Gated(Stability::Unstable, - "rustc_attrs", - "never will be stable", - cfg_fn!(rustc_attrs))), + ( + sym::rustc_args_required_const, + Whitelisted, + template!(List: "N"), + Gated(Stability::Unstable, sym::rustc_attrs, "never will be stable", + cfg_fn!(rustc_attrs)) + ), // RFC 2070 - ("panic_handler", Normal, template!(Word), Ungated), + (sym::panic_handler, Normal, template!(Word), Ungated), - ("alloc_error_handler", Normal, template!(Word), Gated(Stability::Unstable, - "alloc_error_handler", - "#[alloc_error_handler] is an unstable feature", - cfg_fn!(alloc_error_handler))), + (sym::alloc_error_handler, Normal, template!(Word), Gated(Stability::Unstable, + sym::alloc_error_handler, + "`#[alloc_error_handler]` is an unstable feature", + cfg_fn!(alloc_error_handler))), // RFC 2412 - ("optimize", Whitelisted, template!(List: "size|speed"), Gated(Stability::Unstable, - "optimize_attribute", - "#[optimize] attribute is an unstable feature", - cfg_fn!(optimize_attribute))), + (sym::optimize, Whitelisted, template!(List: "size|speed"), Gated(Stability::Unstable, + sym::optimize_attribute, + "`#[optimize]` attribute is an unstable feature", + cfg_fn!(optimize_attribute))), // Crate level attributes - ("crate_name", CrateLevel, template!(NameValueStr: "name"), Ungated), - ("crate_type", CrateLevel, template!(NameValueStr: "bin|lib|..."), Ungated), - ("crate_id", CrateLevel, template!(NameValueStr: "ignored"), Ungated), - ("feature", CrateLevel, template!(List: "name1, name1, ..."), Ungated), - ("no_start", CrateLevel, template!(Word), Ungated), - ("no_main", CrateLevel, template!(Word), Ungated), - ("no_builtins", CrateLevel, template!(Word), Ungated), - ("recursion_limit", CrateLevel, template!(NameValueStr: "N"), Ungated), - ("type_length_limit", CrateLevel, template!(NameValueStr: "N"), Ungated), - ("test_runner", CrateLevel, template!(List: "path"), Gated(Stability::Unstable, - "custom_test_frameworks", - EXPLAIN_CUSTOM_TEST_FRAMEWORKS, + (sym::crate_name, CrateLevel, template!(NameValueStr: "name"), Ungated), + (sym::crate_type, CrateLevel, template!(NameValueStr: "bin|lib|..."), Ungated), + (sym::crate_id, CrateLevel, template!(NameValueStr: "ignored"), Ungated), + (sym::feature, CrateLevel, template!(List: "name1, name1, ..."), Ungated), + (sym::no_start, CrateLevel, template!(Word), Ungated), + (sym::no_main, CrateLevel, template!(Word), Ungated), + (sym::recursion_limit, CrateLevel, template!(NameValueStr: "N"), Ungated), + (sym::type_length_limit, CrateLevel, template!(NameValueStr: "N"), Ungated), + (sym::test_runner, CrateLevel, template!(List: "path"), Gated(Stability::Unstable, + sym::custom_test_frameworks, + "custom test frameworks are an unstable feature", cfg_fn!(custom_test_frameworks))), ]; +pub type BuiltinAttribute = (Symbol, AttributeType, AttributeTemplate, AttributeGate); + +lazy_static! { + pub static ref BUILTIN_ATTRIBUTE_MAP: FxHashMap<Symbol, &'static BuiltinAttribute> = { + let mut map = FxHashMap::default(); + for attr in BUILTIN_ATTRIBUTES.iter() { + if map.insert(attr.0, attr).is_some() { + panic!("duplicate builtin attribute `{}`", attr.0); + } + } + map + }; +} + // cfg(...)'s that are feature gated -const GATED_CFGS: &[(&str, &str, fn(&Features) -> bool)] = &[ +const GATED_CFGS: &[(Symbol, Symbol, fn(&Features) -> bool)] = &[ // (name in cfg, feature, function to check if the feature is enabled) - ("target_thread_local", "cfg_target_thread_local", cfg_fn!(cfg_target_thread_local)), - ("target_has_atomic", "cfg_target_has_atomic", cfg_fn!(cfg_target_has_atomic)), - ("rustdoc", "doc_cfg", cfg_fn!(doc_cfg)), + (sym::target_thread_local, sym::cfg_target_thread_local, cfg_fn!(cfg_target_thread_local)), + (sym::target_has_atomic, sym::cfg_target_has_atomic, cfg_fn!(cfg_target_has_atomic)), + (sym::rustdoc, sym::doc_cfg, cfg_fn!(doc_cfg)), + (sym::doctest, sym::cfg_doctest, cfg_fn!(cfg_doctest)), ]; #[derive(Debug)] @@ -1289,9 +1557,8 @@ pub struct GatedCfg { impl GatedCfg { pub fn gate(cfg: &ast::MetaItem) -> Option<GatedCfg> { - let name = cfg.name().as_str(); GATED_CFGS.iter() - .position(|info| info.0 == name) + .position(|info| cfg.check_name(info.0)) .map(|idx| { GatedCfg { span: cfg.span, @@ -1312,7 +1579,7 @@ impl GatedCfg { struct Context<'a> { features: &'a Features, parse_sess: &'a ParseSess, - plugin_attributes: &'a [(String, AttributeType)], + plugin_attributes: &'a [(Symbol, AttributeType)], } macro_rules! gate_feature_fn { @@ -1331,41 +1598,51 @@ macro_rules! gate_feature_fn { macro_rules! gate_feature { ($cx: expr, $feature: ident, $span: expr, $explain: expr) => { gate_feature_fn!($cx, |x:&Features| x.$feature, $span, - stringify!($feature), $explain, GateStrength::Hard) + sym::$feature, $explain, GateStrength::Hard) }; ($cx: expr, $feature: ident, $span: expr, $explain: expr, $level: expr) => { gate_feature_fn!($cx, |x:&Features| x.$feature, $span, - stringify!($feature), $explain, $level) + sym::$feature, $explain, $level) }; } impl<'a> Context<'a> { - fn check_attribute(&self, attr: &ast::Attribute, is_macro: bool) { + fn check_attribute( + &self, + attr: &ast::Attribute, + attr_info: Option<&BuiltinAttribute>, + is_macro: bool + ) { debug!("check_attribute(attr = {:?})", attr); - let name = attr.name().as_str(); - for &(n, ty, _template, ref gateage) in BUILTIN_ATTRIBUTES { - if name == n { - if let Gated(_, name, desc, ref has_feature) = *gateage { - if !attr.span.allows_unstable(name) { - gate_feature_fn!( - self, has_feature, attr.span, name, desc, GateStrength::Hard + if let Some(&(name, ty, _template, ref gateage)) = attr_info { + if let Gated(_, name, desc, ref has_feature) = *gateage { + if !attr.span.allows_unstable(name) { + gate_feature_fn!( + self, has_feature, attr.span, name, desc, GateStrength::Hard + ); + } + } else if name == sym::doc { + if let Some(content) = attr.meta_item_list() { + if content.iter().any(|c| c.check_name(sym::include)) { + gate_feature!(self, external_doc, attr.span, + "`#[doc(include = \"...\")]` is experimental" ); } - } else if name == "doc" { - if let Some(content) = attr.meta_item_list() { - if content.iter().any(|c| c.check_name("include")) { - gate_feature!(self, external_doc, attr.span, - "#[doc(include = \"...\")] is experimental" - ); - } - } } - debug!("check_attribute: {:?} is builtin, {:?}, {:?}", attr.path, ty, gateage); - return; + } + debug!("check_attribute: {:?} is builtin, {:?}, {:?}", attr.path, ty, gateage); + return; + } else { + for segment in &attr.path.segments { + if segment.ident.as_str().starts_with("rustc") { + let msg = "attributes starting with `rustc` are \ + reserved for use by the `rustc` compiler"; + gate_feature!(self, rustc_attrs, segment.ident.span, msg); + } } } - for &(ref n, ref ty) in self.plugin_attributes { - if attr.path == &**n { + for &(n, ty) in self.plugin_attributes { + if attr.path == n { // Plugins can't gate attributes, so we don't check for it // unlike the code above; we only use this loop to // short-circuit to avoid the checks below. @@ -1373,29 +1650,27 @@ impl<'a> Context<'a> { return; } } - if !attr::is_known(attr) { - if name.starts_with("rustc_") { - let msg = "unless otherwise specified, attributes with the prefix `rustc_` \ - are reserved for internal compiler diagnostics"; - gate_feature!(self, rustc_attrs, attr.span, msg); - } else if !is_macro { - // Only run the custom attribute lint during regular feature gate - // checking. Macro gating runs before the plugin attributes are - // registered, so we skip this in that case. - let msg = format!("The attribute `{}` is currently unknown to the compiler and \ - may have meaning added to it in the future", attr.path); - gate_feature!(self, custom_attribute, attr.span, &msg); - } + if !is_macro && !attr::is_known(attr) { + // Only run the custom attribute lint during regular feature gate + // checking. Macro gating runs before the plugin attributes are + // registered, so we skip this in that case. + let msg = format!("the attribute `{}` is currently unknown to the compiler and \ + may have meaning added to it in the future", attr.path); + gate_feature!(self, custom_attribute, attr.span, &msg); } } } pub fn check_attribute(attr: &ast::Attribute, parse_sess: &ParseSess, features: &Features) { - let cx = Context { features: features, parse_sess: parse_sess, plugin_attributes: &[] }; - cx.check_attribute(attr, true); + let cx = Context { features, parse_sess, plugin_attributes: &[] }; + cx.check_attribute( + attr, + attr.ident().and_then(|ident| BUILTIN_ATTRIBUTE_MAP.get(&ident.name).map(|a| *a)), + true + ); } -fn find_lang_feature_issue(feature: &str) -> Option<u32> { +fn find_lang_feature_issue(feature: Symbol) -> Option<u32> { if let Some(info) = ACTIVE_FEATURES.iter().find(|t| t.0 == feature) { let issue = info.2; // FIXME (#28244): enforce that active features have issue numbers @@ -1425,18 +1700,34 @@ pub enum GateStrength { Soft, } -pub fn emit_feature_err(sess: &ParseSess, feature: &str, span: Span, issue: GateIssue, - explain: &str) { +pub fn emit_feature_err( + sess: &ParseSess, + feature: Symbol, + span: Span, + issue: GateIssue, + explain: &str, +) { feature_err(sess, feature, span, issue, explain).emit(); } -pub fn feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue: GateIssue, - explain: &str) -> DiagnosticBuilder<'a> { +pub fn feature_err<'a, S: Into<MultiSpan>>( + sess: &'a ParseSess, + feature: Symbol, + span: S, + issue: GateIssue, + explain: &str, +) -> DiagnosticBuilder<'a> { leveled_feature_err(sess, feature, span, issue, explain, GateStrength::Hard) } -fn leveled_feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue: GateIssue, - explain: &str, level: GateStrength) -> DiagnosticBuilder<'a> { +fn leveled_feature_err<'a, S: Into<MultiSpan>>( + sess: &'a ParseSess, + feature: Symbol, + span: S, + issue: GateIssue, + explain: &str, + level: GateStrength, +) -> DiagnosticBuilder<'a> { let diag = &sess.span_diagnostic; let issue = match issue { @@ -1444,23 +1735,26 @@ fn leveled_feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue GateIssue::Library(lib) => lib, }; - let explanation = match issue { - None | Some(0) => explain.to_owned(), - Some(n) => format!("{} (see issue #{})", explain, n) - }; - let mut err = match level { GateStrength::Hard => { - diag.struct_span_err_with_code(span, &explanation, stringify_error_code!(E0658)) + diag.struct_span_err_with_code(span, explain, stringify_error_code!(E0658)) } - GateStrength::Soft => diag.struct_span_warn(span, &explanation), + GateStrength::Soft => diag.struct_span_warn(span, explain), }; + match issue { + None | Some(0) => {} // We still accept `0` as a stand-in for backwards compatibility + Some(n) => { + err.note(&format!( + "for more information, see https://github.com/rust-lang/rust/issues/{}", + n, + )); + } + } + // #23973: do not suggest `#![feature(...)]` if we are in beta/stable if sess.unstable_features.is_nightly_build() { - err.help(&format!("add #![feature({})] to the \ - crate attributes to enable", - feature)); + err.help(&format!("add `#![feature({})]` to the crate attributes to enable", feature)); } // If we're on stable and only emitting a "soft" warning, add a note to @@ -1475,31 +1769,11 @@ fn leveled_feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue } const EXPLAIN_BOX_SYNTAX: &str = - "box expression syntax is experimental; you can call `Box::new` instead."; + "box expression syntax is experimental; you can call `Box::new` instead"; pub const EXPLAIN_STMT_ATTR_SYNTAX: &str = - "attributes on expressions are experimental."; - -pub const EXPLAIN_ASM: &str = - "inline assembly is not stable enough for use and is subject to change"; - -pub const EXPLAIN_GLOBAL_ASM: &str = - "`global_asm!` is not stable enough for use and is subject to change"; - -pub const EXPLAIN_CUSTOM_TEST_FRAMEWORKS: &str = - "custom test frameworks are an unstable feature"; - -pub const EXPLAIN_LOG_SYNTAX: &str = - "`log_syntax!` is not stable enough for use and is subject to change"; - -pub const EXPLAIN_CONCAT_IDENTS: &str = - "`concat_idents` is not stable enough for use and is subject to change"; - -pub const EXPLAIN_FORMAT_ARGS_NL: &str = - "`format_args_nl` is only for internal language use and is subject to change"; + "attributes on expressions are experimental"; -pub const EXPLAIN_TRACE_MACROS: &str = - "`trace_macros` is not stable enough for use and is subject to change"; pub const EXPLAIN_ALLOW_INTERNAL_UNSTABLE: &str = "allow_internal_unstable side-steps feature gating and stability checks"; pub const EXPLAIN_ALLOW_INTERNAL_UNSAFE: &str = @@ -1510,18 +1784,19 @@ pub const EXPLAIN_UNSIZED_TUPLE_COERCION: &str = struct PostExpansionVisitor<'a> { context: &'a Context<'a>, + builtin_attributes: &'static FxHashMap<Symbol, &'static BuiltinAttribute>, } macro_rules! gate_feature_post { ($cx: expr, $feature: ident, $span: expr, $explain: expr) => {{ let (cx, span) = ($cx, $span); - if !span.allows_unstable(stringify!($feature)) { + if !span.allows_unstable(sym::$feature) { gate_feature!(cx.context, $feature, span, $explain) } }}; ($cx: expr, $feature: ident, $span: expr, $explain: expr, $level: expr) => {{ let (cx, span) = ($cx, $span); - if !span.allows_unstable(stringify!($feature)) { + if !span.allows_unstable(sym::$feature) { gate_feature!(cx.context, $feature, span, $explain, $level) } }} @@ -1582,139 +1857,96 @@ impl<'a> PostExpansionVisitor<'a> { Abi::System => {} } } - - fn check_builtin_attribute(&mut self, attr: &ast::Attribute, name: &str, - template: AttributeTemplate) { - // Some special attributes like `cfg` must be checked - // before the generic check, so we skip them here. - let should_skip = |name| name == "cfg"; - // Some of previously accepted forms were used in practice, - // report them as warnings for now. - let should_warn = |name| name == "doc" || name == "ignore" || - name == "inline" || name == "link"; - - match attr.parse_meta(self.context.parse_sess) { - Ok(meta) => if !should_skip(name) && !template.compatible(&meta.node) { - let mut msg = "attribute must be of the form ".to_owned(); - let mut first = true; - if template.word { - first = false; - msg.push_str(&format!("`#[{}{}]`", name, "")); - } - if let Some(descr) = template.list { - if !first { - msg.push_str(" or "); - } - first = false; - msg.push_str(&format!("`#[{}({})]`", name, descr)); - } - if let Some(descr) = template.name_value_str { - if !first { - msg.push_str(" or "); - } - msg.push_str(&format!("`#[{} = \"{}\"]`", name, descr)); - } - if should_warn(name) { - self.context.parse_sess.buffer_lint( - BufferedEarlyLintId::IllFormedAttributeInput, - meta.span, - ast::CRATE_NODE_ID, - &msg, - ); - } else { - self.context.parse_sess.span_diagnostic.span_err(meta.span, &msg); - } - } - Err(mut err) => err.emit(), - } - } } impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { fn visit_attribute(&mut self, attr: &ast::Attribute) { - // check for gated attributes - self.context.check_attribute(attr, false); + let attr_info = attr.ident().and_then(|ident| { + self.builtin_attributes.get(&ident.name).map(|a| *a) + }); - if attr.check_name("doc") { + // Check for gated attributes. + self.context.check_attribute(attr, attr_info, false); + + if attr.check_name(sym::doc) { if let Some(content) = attr.meta_item_list() { - if content.len() == 1 && content[0].check_name("cfg") { + if content.len() == 1 && content[0].check_name(sym::cfg) { gate_feature_post!(&self, doc_cfg, attr.span, - "#[doc(cfg(...))] is experimental" + "`#[doc(cfg(...))]` is experimental" ); - } else if content.iter().any(|c| c.check_name("masked")) { + } else if content.iter().any(|c| c.check_name(sym::masked)) { gate_feature_post!(&self, doc_masked, attr.span, - "#[doc(masked)] is experimental" + "`#[doc(masked)]` is experimental" ); - } else if content.iter().any(|c| c.check_name("spotlight")) { + } else if content.iter().any(|c| c.check_name(sym::spotlight)) { gate_feature_post!(&self, doc_spotlight, attr.span, - "#[doc(spotlight)] is experimental" + "`#[doc(spotlight)]` is experimental" ); - } else if content.iter().any(|c| c.check_name("alias")) { + } else if content.iter().any(|c| c.check_name(sym::alias)) { gate_feature_post!(&self, doc_alias, attr.span, - "#[doc(alias = \"...\")] is experimental" + "`#[doc(alias = \"...\")]` is experimental" ); - } else if content.iter().any(|c| c.check_name("keyword")) { + } else if content.iter().any(|c| c.check_name(sym::keyword)) { gate_feature_post!(&self, doc_keyword, attr.span, - "#[doc(keyword = \"...\")] is experimental" + "`#[doc(keyword = \"...\")]` is experimental" ); } } } - match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == name) { - Some(&(name, _, template, _)) => self.check_builtin_attribute(attr, name, template), - None => if let Some(TokenTree::Token(_, token::Eq)) = attr.tokens.trees().next() { - // All key-value attributes are restricted to meta-item syntax. - attr.parse_meta(self.context.parse_sess).map_err(|mut err| err.emit()).ok(); + match attr_info { + // `rustc_dummy` doesn't have any restrictions specific to built-in attributes. + Some(&(name, _, template, _)) if name != sym::rustc_dummy => + check_builtin_attribute(self.context.parse_sess, attr, name, template), + _ => if let Some(TokenTree::Token(token)) = attr.tokens.trees().next() { + if token == token::Eq { + // All key-value attributes are restricted to meta-item syntax. + attr.parse_meta(self.context.parse_sess).map_err(|mut err| err.emit()).ok(); + } } } } fn visit_name(&mut self, sp: Span, name: ast::Name) { if !name.as_str().is_ascii() { - gate_feature_post!(&self, - non_ascii_idents, - self.context.parse_sess.source_map().def_span(sp), - "non-ascii idents are not fully supported."); + gate_feature_post!( + &self, + non_ascii_idents, + self.context.parse_sess.source_map().def_span(sp), + "non-ascii idents are not fully supported" + ); } } fn visit_item(&mut self, i: &'a ast::Item) { match i.node { - ast::ItemKind::Const(_,_) => { - if i.ident.name == "_" { - gate_feature_post!(&self, underscore_const_names, i.span, - "naming constants with `_` is unstable"); - } - } - ast::ItemKind::ForeignMod(ref foreign_module) => { self.check_abi(foreign_module.abi, i.span); } ast::ItemKind::Fn(..) => { - if attr::contains_name(&i.attrs[..], "plugin_registrar") { + if attr::contains_name(&i.attrs[..], sym::plugin_registrar) { gate_feature_post!(&self, plugin_registrar, i.span, "compiler plugins are experimental and possibly buggy"); } - if attr::contains_name(&i.attrs[..], "start") { + if attr::contains_name(&i.attrs[..], sym::start) { gate_feature_post!(&self, start, i.span, - "a #[start] function is an experimental \ + "a `#[start]` function is an experimental \ feature whose signature may change \ over time"); } - if attr::contains_name(&i.attrs[..], "main") { + if attr::contains_name(&i.attrs[..], sym::main) { gate_feature_post!(&self, main, i.span, - "declaration of a nonstandard #[main] \ + "declaration of a non-standard `#[main]` \ function may change over time, for now \ a top-level `fn main()` is required"); } } ast::ItemKind::Struct(..) => { - for attr in attr::filter_by_name(&i.attrs[..], "repr") { + for attr in attr::filter_by_name(&i.attrs[..], sym::repr) { for item in attr.meta_item_list().unwrap_or_else(Vec::new) { - if item.check_name("simd") { + if item.check_name(sym::simd) { gate_feature_post!(&self, repr_simd, attr.span, "SIMD types are experimental and possibly buggy"); } @@ -1722,15 +1954,27 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } } - ast::ItemKind::Enum(..) => { - for attr in attr::filter_by_name(&i.attrs[..], "repr") { - for item in attr.meta_item_list().unwrap_or_else(Vec::new) { - if item.check_name("align") { - gate_feature_post!(&self, repr_align_enum, attr.span, - "`#[repr(align(x))]` on enums is experimental"); - } + ast::ItemKind::Enum(ast::EnumDef{ref variants, ..}, ..) => { + for variant in variants { + match (&variant.node.data, &variant.node.disr_expr) { + (ast::VariantData::Unit(..), _) => {}, + (_, Some(disr_expr)) => + gate_feature_post!( + &self, + arbitrary_enum_discriminant, + disr_expr.value.span, + "discriminants on non-unit variants are experimental"), + _ => {}, } } + + let has_feature = self.context.features.arbitrary_enum_discriminant; + if !has_feature && !i.span.allows_unstable(sym::arbitrary_enum_discriminant) { + Parser::maybe_report_invalid_custom_discriminants( + self.context.parse_sess, + &variants, + ); + } } ast::ItemKind::Impl(_, polarity, defaultness, _, _, _, _) => { @@ -1768,12 +2012,12 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { gate_feature_post!(&self, decl_macro, i.span, msg); } - ast::ItemKind::Existential(..) => { + ast::ItemKind::OpaqueTy(..) => { gate_feature_post!( &self, - existential_type, + type_alias_impl_trait, i.span, - "existential types are unstable" + "`impl Trait` in type aliases is unstable" ); } @@ -1787,7 +2031,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { match i.node { ast::ForeignItemKind::Fn(..) | ast::ForeignItemKind::Static(..) => { - let link_name = attr::first_attr_value_str_by_name(&i.attrs, "link_name"); + let link_name = attr::first_attr_value_str_by_name(&i.attrs, sym::link_name); let links_to_llvm = match link_name { Some(val) => val.as_str().starts_with("llvm."), _ => false @@ -1824,7 +2068,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { fn visit_fn_ret_ty(&mut self, ret_ty: &'a ast::FunctionRetTy) { if let ast::FunctionRetTy::Ty(ref output_ty) = *ret_ty { if let ast::TyKind::Never = output_ty.node { - // Do nothing + // Do nothing. } else { self.visit_ty(output_ty) } @@ -1837,11 +2081,12 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { gate_feature_post!(&self, box_syntax, e.span, EXPLAIN_BOX_SYNTAX); } ast::ExprKind::Type(..) => { - gate_feature_post!(&self, type_ascription, e.span, - "type ascription is experimental"); - } - ast::ExprKind::ObsoleteInPlace(..) => { - // these get a hard error in ast-validation + // To avoid noise about type ascription in common syntax errors, only emit if it + // is the *only* error. + if self.context.parse_sess.span_diagnostic.err_count() == 0 { + gate_feature_post!(&self, type_ascription, e.span, + "type ascription is experimental"); + } } ast::ExprKind::Yield(..) => { gate_feature_post!(&self, generators, @@ -1857,15 +2102,15 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { "labels on blocks are unstable"); } } - ast::ExprKind::Closure(_, ast::IsAsync::Async { .. }, ..) => { - gate_feature_post!(&self, async_await, e.span, "async closures are unstable"); - } ast::ExprKind::Async(..) => { gate_feature_post!(&self, async_await, e.span, "async blocks are unstable"); } + ast::ExprKind::Await(_) => { + gate_feature_post!(&self, async_await, e.span, "async/await is unstable"); + } _ => {} } - visit::walk_expr(self, e); + visit::walk_expr(self, e) } fn visit_arm(&mut self, arm: &'a ast::Arm) { @@ -1873,11 +2118,23 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } fn visit_pat(&mut self, pattern: &'a ast::Pat) { - match pattern.node { - PatKind::Slice(_, Some(ref subslice), _) => { - gate_feature_post!(&self, slice_patterns, - subslice.span, - "syntax for subslices in slice patterns is not yet stabilized"); + match &pattern.node { + PatKind::Slice(pats) => { + for pat in &*pats { + let span = pat.span; + let inner_pat = match &pat.node { + PatKind::Ident(.., Some(pat)) => pat, + _ => pat, + }; + if inner_pat.is_rest() { + gate_feature_post!( + &self, + slice_patterns, + span, + "subslice patterns are unstable" + ); + } + } } PatKind::Box(..) => { gate_feature_post!(&self, box_patterns, @@ -1898,37 +2155,43 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { fn_decl: &'a ast::FnDecl, span: Span, _node_id: NodeId) { - match fn_kind { - FnKind::ItemFn(_, header, _, _) => { - // Check for const fn and async fn declarations. - if header.asyncness.node.is_async() { - gate_feature_post!(&self, async_await, span, "async fn is unstable"); - } + if let Some(header) = fn_kind.header() { + // Check for const fn and async fn declarations. + if header.asyncness.node.is_async() { + gate_feature_post!(&self, async_await, span, "async fn is unstable"); + } - if fn_decl.c_variadic { - gate_feature_post!(&self, c_variadic, span, - "C-varaidic functions are unstable"); - } - // Stability of const fn methods are covered in - // `visit_trait_item` and `visit_impl_item` below; this is - // because default methods don't pass through this point. + // Stability of const fn methods are covered in + // `visit_trait_item` and `visit_impl_item` below; this is + // because default methods don't pass through this point. + self.check_abi(header.abi, span); + } - self.check_abi(header.abi, span); - } - FnKind::Method(_, sig, _, _) => { - self.check_abi(sig.header.abi, span); - } - _ => {} + if fn_decl.c_variadic { + gate_feature_post!(&self, c_variadic, span, "C-variadic functions are unstable"); } - visit::walk_fn(self, fn_kind, fn_decl, span); + + visit::walk_fn(self, fn_kind, fn_decl, span) } fn visit_generic_param(&mut self, param: &'a GenericParam) { - if let GenericParamKind::Const { .. } = param.kind { - gate_feature_post!(&self, const_generics, param.ident.span, - "const generics are unstable"); + match param.kind { + GenericParamKind::Const { .. } => + gate_feature_post!(&self, const_generics, param.ident.span, + "const generics are unstable"), + _ => {} } - visit::walk_generic_param(self, param); + visit::walk_generic_param(self, param) + } + + fn visit_assoc_ty_constraint(&mut self, constraint: &'a AssocTyConstraint) { + match constraint.kind { + AssocTyConstraintKind::Bound { .. } => + gate_feature_post!(&self, associated_type_bounds, constraint.span, + "associated type bounds are unstable"), + _ => {} + } + visit::walk_assoc_ty_constraint(self, constraint) } fn visit_trait_item(&mut self, ti: &'a ast::TraitItem) { @@ -1937,9 +2200,12 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { if block.is_none() { self.check_abi(sig.header.abi, ti.span); } + if sig.header.asyncness.node.is_async() { + gate_feature_post!(&self, async_await, ti.span, "async fn is unstable"); + } if sig.decl.c_variadic { gate_feature_post!(&self, c_variadic, ti.span, - "C-varaidic functions are unstable"); + "C-variadic functions are unstable"); } if sig.header.constness.node == ast::Constness::Const { gate_feature_post!(&self, const_fn, ti.span, "const fn is unstable"); @@ -1963,7 +2229,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } _ => {} } - visit::walk_trait_item(self, ti); + visit::walk_trait_item(self, ti) } fn visit_impl_item(&mut self, ii: &'a ast::ImplItem) { @@ -1975,15 +2241,15 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { match ii.node { ast::ImplItemKind::Method(..) => {} - ast::ImplItemKind::Existential(..) => { + ast::ImplItemKind::OpaqueTy(..) => { gate_feature_post!( &self, - existential_type, + type_alias_impl_trait, ii.span, - "existential types are unstable" + "`impl Trait` in type aliases is unstable" ); } - ast::ImplItemKind::Type(_) => { + ast::ImplItemKind::TyAlias(_) => { if !ii.generics.params.is_empty() { gate_feature_post!(&self, generic_associated_types, ii.span, "generic associated types are unstable"); @@ -1995,7 +2261,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } _ => {} } - visit::walk_impl_item(self, ii); + visit::walk_impl_item(self, ii) } fn visit_vis(&mut self, vis: &'a ast::Visibility) { @@ -2003,25 +2269,22 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { gate_feature_post!(&self, crate_visibility_modifier, vis.span, "`crate` visibility modifier is experimental"); } - visit::walk_vis(self, vis); + visit::walk_vis(self, vis) } } pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], - crate_edition: Edition) -> Features { + crate_edition: Edition, allow_features: &Option<Vec<String>>) -> Features { fn feature_removed(span_handler: &Handler, span: Span, reason: Option<&str>) { let mut err = struct_span_err!(span_handler, span, E0557, "feature has been removed"); if let Some(reason) = reason { err.span_note(span, reason); + } else { + err.span_label(span, "feature has been removed"); } err.emit(); } - // Some features are known to be incomplete and using them is likely to have - // unanticipated results, such as compiler crashes. We warn the user about these - // to alert them. - let incomplete_features = ["generic_associated_types", "const_generics"]; - let mut features = Features::new(); let mut edition_enabled_features = FxHashMap::default(); @@ -2029,7 +2292,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], if edition <= crate_edition { // The `crate_edition` implies its respective umbrella feature-gate // (i.e., `#![feature(rust_20XX_preview)]` isn't needed on edition 20XX). - edition_enabled_features.insert(Symbol::intern(edition.feature_name()), edition); + edition_enabled_features.insert(edition.feature_name(), edition); } } @@ -2037,7 +2300,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], if let Some(f_edition) = f_edition { if f_edition <= crate_edition { set(&mut features, DUMMY_SP); - edition_enabled_features.insert(Symbol::intern(name), crate_edition); + edition_enabled_features.insert(name, crate_edition); } } } @@ -2045,7 +2308,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], // Process the edition umbrella feature-gates first, to ensure // `edition_enabled_features` is completed before it's queried. for attr in krate_attrs { - if !attr.check_name("feature") { + if !attr.check_name(sym::feature) { continue } @@ -2055,22 +2318,12 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], }; for mi in list { - let name = if let Some(word) = mi.word() { - word.name() - } else { - continue - }; - - if incomplete_features.iter().any(|f| *f == name.as_str()) { - span_handler.struct_span_warn( - mi.span, - &format!( - "the feature `{}` is incomplete and may cause the compiler to crash", - name - ) - ).emit(); + if !mi.is_word() { + continue; } + let name = mi.name_or_empty(); + if let Some(edition) = ALL_EDITIONS.iter().find(|e| name == e.feature_name()) { if *edition <= crate_edition { continue; @@ -2082,7 +2335,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], // FIXME(Manishearth) there is currently no way to set // lib features by edition set(&mut features, DUMMY_SP); - edition_enabled_features.insert(Symbol::intern(name), *edition); + edition_enabled_features.insert(name, *edition); } } } @@ -2091,7 +2344,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], } for attr in krate_attrs { - if !attr.check_name("feature") { + if !attr.check_name(sym::feature) { continue } @@ -2100,19 +2353,32 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], None => continue, }; + let bad_input = |span| { + struct_span_err!(span_handler, span, E0556, "malformed `feature` attribute input") + }; + for mi in list { - let name = if let Some(word) = mi.word() { - word.name() - } else { - span_err!(span_handler, mi.span, E0556, - "malformed feature, expected just one word"); - continue + let name = match mi.ident() { + Some(ident) if mi.is_word() => ident.name, + Some(ident) => { + bad_input(mi.span()).span_suggestion( + mi.span(), + "expected just one word", + format!("{}", ident.name), + Applicability::MaybeIncorrect, + ).emit(); + continue + } + None => { + bad_input(mi.span()).span_label(mi.span(), "expected just one word").emit(); + continue + } }; if let Some(edition) = edition_enabled_features.get(&name) { struct_span_warn!( span_handler, - mi.span, + mi.span(), E0705, "the feature `{}` is included in the Rust {} edition", name, @@ -2126,36 +2392,49 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute], continue; } - if let Some((.., set)) = ACTIVE_FEATURES.iter().find(|f| name == f.0) { - set(&mut features, mi.span); - features.declared_lang_features.push((name, mi.span, None)); - continue - } - let removed = REMOVED_FEATURES.iter().find(|f| name == f.0); let stable_removed = STABLE_REMOVED_FEATURES.iter().find(|f| name == f.0); if let Some((.., reason)) = removed.or(stable_removed) { - feature_removed(span_handler, mi.span, *reason); - continue + feature_removed(span_handler, mi.span(), *reason); + continue; } if let Some((_, since, ..)) = ACCEPTED_FEATURES.iter().find(|f| name == f.0) { let since = Some(Symbol::intern(since)); - features.declared_lang_features.push((name, mi.span, since)); - continue + features.declared_lang_features.push((name, mi.span(), since)); + continue; + } + + if let Some(allowed) = allow_features.as_ref() { + if allowed.iter().find(|f| *f == name.as_str()).is_none() { + span_err!(span_handler, mi.span(), E0725, + "the feature `{}` is not in the list of allowed features", + name); + continue; + } } - features.declared_lib_features.push((name, mi.span)); + if let Some((.., set)) = ACTIVE_FEATURES.iter().find(|f| name == f.0) { + set(&mut features, mi.span()); + features.declared_lang_features.push((name, mi.span(), None)); + continue; + } + + features.declared_lib_features.push((name, mi.span())); } } features } +fn for_each_in_lock<T>(vec: &Lock<Vec<T>>, f: impl Fn(&T)) { + vec.borrow().iter().for_each(f); +} + pub fn check_crate(krate: &ast::Crate, sess: &ParseSess, features: &Features, - plugin_attributes: &[(String, AttributeType)], + plugin_attributes: &[(Symbol, AttributeType)], unstable: UnstableFeatures) { maybe_stage_features(&sess.span_diagnostic, krate, unstable); let ctx = Context { @@ -2164,7 +2443,31 @@ pub fn check_crate(krate: &ast::Crate, plugin_attributes, }; - let visitor = &mut PostExpansionVisitor { context: &ctx }; + for_each_in_lock(&sess.param_attr_spans, |span| gate_feature!( + &ctx, + param_attrs, + *span, + "attributes on function parameters are unstable" + )); + + for_each_in_lock(&sess.let_chains_spans, |span| gate_feature!( + &ctx, + let_chains, + *span, + "`let` expressions in this position are experimental" + )); + + for_each_in_lock(&sess.async_closure_spans, |span| gate_feature!( + &ctx, + async_closure, + *span, + "async closures are unstable" + )); + + let visitor = &mut PostExpansionVisitor { + context: &ctx, + builtin_attributes: &*BUILTIN_ATTRIBUTE_MAP, + }; visit::walk_crate(visitor, krate); } @@ -2211,10 +2514,10 @@ fn maybe_stage_features(span_handler: &Handler, krate: &ast::Crate, }; if !allow_features { for attr in &krate.attrs { - if attr.check_name("feature") { + if attr.check_name(sym::feature) { let release_channel = option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)"); span_err!(span_handler, attr.span, E0554, - "#![feature] may not be used on the {} release channel", + "`#![feature]` may not be used on the {} release channel", release_channel); } } diff --git a/src/libsyntax/json.rs b/src/libsyntax/json.rs index 9acd0d099a0..83c9c692bd3 100644 --- a/src/libsyntax/json.rs +++ b/src/libsyntax/json.rs @@ -14,11 +14,12 @@ use crate::source_map::{SourceMap, FilePathMapping}; use errors::registry::Registry; use errors::{DiagnosticBuilder, SubDiagnostic, CodeSuggestion, SourceMapper}; use errors::{DiagnosticId, Applicability}; -use errors::emitter::{Emitter, EmitterWriter}; +use errors::emitter::{Emitter, HumanReadableErrorType}; use syntax_pos::{MacroBacktrace, Span, SpanLabel, MultiSpan}; use rustc_data_structures::sync::{self, Lrc}; use std::io::{self, Write}; +use std::path::Path; use std::vec; use std::sync::{Arc, Mutex}; @@ -30,37 +31,46 @@ pub struct JsonEmitter { sm: Lrc<dyn SourceMapper + sync::Send + sync::Sync>, pretty: bool, ui_testing: bool, + json_rendered: HumanReadableErrorType, } impl JsonEmitter { - pub fn stderr(registry: Option<Registry>, - source_map: Lrc<SourceMap>, - pretty: bool) -> JsonEmitter { + pub fn stderr( + registry: Option<Registry>, + source_map: Lrc<SourceMap>, + pretty: bool, + json_rendered: HumanReadableErrorType, + ) -> JsonEmitter { JsonEmitter { dst: Box::new(io::stderr()), registry, sm: source_map, pretty, ui_testing: false, + json_rendered, } } - pub fn basic(pretty: bool) -> JsonEmitter { + pub fn basic(pretty: bool, json_rendered: HumanReadableErrorType) -> JsonEmitter { let file_path_mapping = FilePathMapping::empty(); JsonEmitter::stderr(None, Lrc::new(SourceMap::new(file_path_mapping)), - pretty) + pretty, json_rendered) } - pub fn new(dst: Box<dyn Write + Send>, - registry: Option<Registry>, - source_map: Lrc<SourceMap>, - pretty: bool) -> JsonEmitter { + pub fn new( + dst: Box<dyn Write + Send>, + registry: Option<Registry>, + source_map: Lrc<SourceMap>, + pretty: bool, + json_rendered: HumanReadableErrorType, + ) -> JsonEmitter { JsonEmitter { dst, registry, sm: source_map, pretty, ui_testing: false, + json_rendered, } } @@ -70,7 +80,7 @@ impl JsonEmitter { } impl Emitter for JsonEmitter { - fn emit(&mut self, db: &DiagnosticBuilder<'_>) { + fn emit_diagnostic(&mut self, db: &DiagnosticBuilder<'_>) { let data = Diagnostic::from_diagnostic_builder(db, self); let result = if self.pretty { writeln!(&mut self.dst, "{}", as_pretty_json(&data)) @@ -81,6 +91,18 @@ impl Emitter for JsonEmitter { panic!("failed to print diagnostics: {:?}", e); } } + + fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) { + let data = ArtifactNotification { artifact: path, emit: artifact_type }; + let result = if self.pretty { + writeln!(&mut self.dst, "{}", as_pretty_json(&data)) + } else { + writeln!(&mut self.dst, "{}", as_json(&data)) + }; + if let Err(e) = result { + panic!("failed to print notification: {:?}", e); + } + } } // The following data types are provided just for serialisation. @@ -100,7 +122,6 @@ struct Diagnostic { } #[derive(RustcEncodable)] -#[allow(unused_attributes)] struct DiagnosticSpan { file_name: String, byte_start: u32, @@ -148,7 +169,7 @@ struct DiagnosticSpanMacroExpansion { macro_decl_name: String, /// span where macro was defined (if known) - def_site_span: Option<DiagnosticSpan>, + def_site_span: DiagnosticSpan, } #[derive(RustcEncodable)] @@ -159,6 +180,14 @@ struct DiagnosticCode { explanation: Option<&'static str>, } +#[derive(RustcEncodable)] +struct ArtifactNotification<'a> { + /// The path of the artifact. + artifact: &'a Path, + /// What kind of artifact we're emitting. + emit: &'a str, +} + impl Diagnostic { fn from_diagnostic_builder(db: &DiagnosticBuilder<'_>, je: &JsonEmitter) @@ -190,8 +219,8 @@ impl Diagnostic { } let buf = BufWriter::default(); let output = buf.clone(); - EmitterWriter::new(Box::new(buf), Some(je.sm.clone()), false, false) - .ui_testing(je.ui_testing).emit(db); + je.json_rendered.new_emitter(Box::new(buf), Some(je.sm.clone()), false) + .ui_testing(je.ui_testing).emit_diagnostic(db); let output = Arc::try_unwrap(output.0).unwrap().into_inner().unwrap(); let output = String::from_utf8(output).unwrap(); @@ -270,14 +299,13 @@ impl DiagnosticSpan { None, backtrace, je); - let def_site_span = bt.def_site_span.map(|sp| { - Self::from_span_full(sp, + let def_site_span = + Self::from_span_full(bt.def_site_span, false, None, None, vec![].into_iter(), - je) - }); + je); Box::new(DiagnosticSpanMacroExpansion { span: call_site, macro_decl_name: bt.macro_decl_name, @@ -348,19 +376,17 @@ impl DiagnosticSpanLine { /// `span` within the line. fn from_span(span: Span, je: &JsonEmitter) -> Vec<DiagnosticSpanLine> { je.sm.span_to_lines(span) - .map(|lines| { - let fm = &*lines.file; - lines.lines - .iter() - .map(|line| { - DiagnosticSpanLine::line_from_source_file(fm, - line.line_index, - line.start_col.0 + 1, - line.end_col.0 + 1) - }) - .collect() - }) - .unwrap_or_else(|_| vec![]) + .map(|lines| { + let fm = &*lines.file; + lines.lines + .iter() + .map(|line| DiagnosticSpanLine::line_from_source_file( + fm, + line.line_index, + line.start_col.0 + 1, + line.end_col.0 + 1, + )).collect() + }).unwrap_or_else(|_| vec![]) } } diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index a6145d5dcb3..8ac48d8d74a 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -7,32 +7,42 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))] -#![deny(rust_2018_idioms)] - +#![feature(bind_by_move_pattern_guards)] +#![feature(box_syntax)] +#![feature(const_fn)] +#![feature(const_transmute)] #![feature(crate_visibility_modifier)] #![feature(label_break_value)] +#![feature(mem_take)] #![feature(nll)] -#![feature(rustc_attrs)] +#![feature(proc_macro_diagnostic)] +#![feature(proc_macro_internals)] +#![feature(proc_macro_span)] #![feature(rustc_diagnostic_macros)] -#![feature(step_trait)] #![feature(try_trait)] #![feature(unicode_internals)] #![recursion_limit="256"] -#[allow(unused_extern_crates)] -extern crate serialize as rustc_serialize; // used by deriving +extern crate proc_macro; pub use errors; use rustc_data_structures::sync::Lock; use rustc_data_structures::bit_set::GrowableBitSet; pub use rustc_data_structures::thin_vec::ThinVec; use ast::AttrId; +use syntax_pos::edition::Edition; + +#[cfg(test)] +mod tests; + +const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); // A variant of 'try!' that panics on an Err. This is used as a crutch on the // way towards a non-panic!-prone parser. It should be used for fatal parsing // errors; eventually we plan to convert all code using panictry to just use // normal try. +#[macro_export] macro_rules! panictry { ($e:expr) => ({ use std::result::Result::{Ok, Err}; @@ -81,26 +91,32 @@ pub struct Globals { } impl Globals { - fn new() -> Globals { + fn new(edition: Edition) -> Globals { Globals { // We have no idea how many attributes their will be, so just // initiate the vectors with 0 bits. We'll grow them as necessary. used_attrs: Lock::new(GrowableBitSet::new_empty()), known_attrs: Lock::new(GrowableBitSet::new_empty()), - syntax_pos_globals: syntax_pos::Globals::new(), + syntax_pos_globals: syntax_pos::Globals::new(edition), } } } -pub fn with_globals<F, R>(f: F) -> R +pub fn with_globals<F, R>(edition: Edition, f: F) -> R where F: FnOnce() -> R { - let globals = Globals::new(); + let globals = Globals::new(edition); GLOBALS.set(&globals, || { syntax_pos::GLOBALS.set(&globals.syntax_pos_globals, f) }) } +pub fn with_default_globals<F, R>(f: F) -> R + where F: FnOnce() -> R +{ + with_globals(edition::DEFAULT_EDITION, f) +} + scoped_tls::scoped_thread_local!(pub static GLOBALS: Globals); #[macro_use] @@ -113,25 +129,17 @@ pub mod diagnostics { // N.B., this module needs to be declared first so diagnostics are // registered before they are used. -pub mod diagnostic_list; +pub mod error_codes; pub mod util { pub mod lev_distance; pub mod node_count; pub mod parser; - #[cfg(test)] - pub mod parser_testing; pub mod map_in_place; } pub mod json; -pub mod syntax { - pub use crate::ext; - pub use crate::parse; - pub use crate::ast; -} - pub mod ast; pub mod attr; pub mod source_map; @@ -143,29 +151,31 @@ pub mod mut_visit; pub mod parse; pub mod ptr; pub mod show_span; -pub mod std_inject; pub use syntax_pos::edition; pub use syntax_pos::symbol; -pub mod test; pub mod tokenstream; pub mod visit; pub mod print { pub mod pp; pub mod pprust; + mod helpers; } pub mod ext { + mod placeholders; + mod proc_macro_server; + pub use syntax_pos::hygiene; + pub mod allocator; pub mod base; pub mod build; - pub mod derive; pub mod expand; - pub mod placeholders; - pub mod source_util; + pub mod proc_macro; pub mod tt { pub mod transcribe; + pub mod macro_check; pub mod macro_parser; pub mod macro_rules; pub mod quoted; @@ -174,7 +184,4 @@ pub mod ext { pub mod early_buffered_lints; -#[cfg(test)] -mod test_snippet; - __build_diagnostic_array! { libsyntax, DIAGNOSTICS } diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 802b7808695..be04c6a76b0 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -11,7 +11,6 @@ use crate::ast::*; use crate::source_map::{Spanned, respan}; use crate::parse::token::{self, Token}; use crate::ptr::P; -use crate::symbol::keywords; use crate::ThinVec; use crate::tokenstream::*; use crate::util::map_in_place::MapInPlace; @@ -23,6 +22,9 @@ use rustc_data_structures::sync::Lrc; use std::ops::DerefMut; use std::{panic, process, ptr}; +#[cfg(test)] +mod tests; + pub trait ExpectOne<A: Array> { fn expect_one(self, err: &'static str) -> A::Item; } @@ -132,10 +134,6 @@ pub trait MutVisitor: Sized { noop_visit_arm(a, self); } - fn visit_guard(&mut self, g: &mut Guard) { - noop_visit_guard(g, self); - } - fn visit_pat(&mut self, p: &mut P<Pat>) { noop_visit_pat(p, self); } @@ -164,8 +162,8 @@ pub trait MutVisitor: Sized { noop_visit_lifetime(l, self); } - fn visit_ty_binding(&mut self, t: &mut TypeBinding) { - noop_visit_ty_binding(t, self); + fn visit_ty_constraint(&mut self, t: &mut AssocTyConstraint) { + noop_visit_ty_constraint(t, self); } fn visit_mod(&mut self, m: &mut Mod) { @@ -384,24 +382,31 @@ pub fn noop_visit_use_tree<T: MutVisitor>(use_tree: &mut UseTree, vis: &mut T) { vis.visit_span(span); } -pub fn noop_visit_arm<T: MutVisitor>(Arm { attrs, pats, guard, body }: &mut Arm, vis: &mut T) { +pub fn noop_visit_arm<T: MutVisitor>( + Arm { attrs, pats, guard, body, span }: &mut Arm, + vis: &mut T, +) { visit_attrs(attrs, vis); visit_vec(pats, |pat| vis.visit_pat(pat)); - visit_opt(guard, |guard| vis.visit_guard(guard)); + visit_opt(guard, |guard| vis.visit_expr(guard)); vis.visit_expr(body); + vis.visit_span(span); } -pub fn noop_visit_guard<T: MutVisitor>(g: &mut Guard, vis: &mut T) { - match g { - Guard::If(e) => vis.visit_expr(e), - } -} - -pub fn noop_visit_ty_binding<T: MutVisitor>(TypeBinding { id, ident, ty, span }: &mut TypeBinding, - vis: &mut T) { +pub fn noop_visit_ty_constraint<T: MutVisitor>( + AssocTyConstraint { id, ident, kind, span }: &mut AssocTyConstraint, + vis: &mut T +) { vis.visit_id(id); vis.visit_ident(ident); - vis.visit_ty(ty); + match kind { + AssocTyConstraintKind::Equality { ref mut ty } => { + vis.visit_ty(ty); + } + AssocTyConstraintKind::Bound { ref mut bounds } => { + visit_bounds(bounds, vis); + } + } vis.visit_span(span); } @@ -450,9 +455,10 @@ pub fn noop_visit_foreign_mod<T: MutVisitor>(foreign_mod: &mut ForeignMod, vis: } pub fn noop_visit_variant<T: MutVisitor>(variant: &mut Variant, vis: &mut T) { - let Spanned { node: Variant_ { ident, attrs, data, disr_expr }, span } = variant; + let Spanned { node: Variant_ { ident, attrs, id, data, disr_expr }, span } = variant; vis.visit_ident(ident); visit_attrs(attrs, vis); + vis.visit_id(id); vis.visit_variant_data(data); visit_opt(disr_expr, |disr_expr| vis.visit_anon_const(disr_expr)); vis.visit_span(span); @@ -495,9 +501,9 @@ pub fn noop_visit_generic_arg<T: MutVisitor>(arg: &mut GenericArg, vis: &mut T) pub fn noop_visit_angle_bracketed_parameter_data<T: MutVisitor>(data: &mut AngleBracketedArgs, vis: &mut T) { - let AngleBracketedArgs { args, bindings, span } = data; + let AngleBracketedArgs { args, constraints, span } = data; visit_vec(args, |arg| vis.visit_generic_arg(arg)); - visit_vec(bindings, |binding| vis.visit_ty_binding(binding)); + visit_vec(constraints, |constraint| vis.visit_ty_constraint(constraint)); vis.visit_span(span); } @@ -527,7 +533,7 @@ pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) { } pub fn noop_visit_mac<T: MutVisitor>(Spanned { node, span }: &mut Mac, vis: &mut T) { - let Mac_ { path, delim: _, tts } = node; + let Mac_ { path, delim: _, tts, .. } = node; vis.visit_path(path); vis.visit_tts(tts); vis.visit_span(span); @@ -539,16 +545,14 @@ pub fn noop_visit_macro_def<T: MutVisitor>(macro_def: &mut MacroDef, vis: &mut T } pub fn noop_visit_meta_list_item<T: MutVisitor>(li: &mut NestedMetaItem, vis: &mut T) { - let Spanned { node, span } = li; - match node { - NestedMetaItemKind::MetaItem(mi) => vis.visit_meta_item(mi), - NestedMetaItemKind::Literal(_lit) => {} + match li { + NestedMetaItem::MetaItem(mi) => vis.visit_meta_item(mi), + NestedMetaItem::Literal(_lit) => {} } - vis.visit_span(span); } pub fn noop_visit_meta_item<T: MutVisitor>(mi: &mut MetaItem, vis: &mut T) { - let MetaItem { ident: _, node, span } = mi; + let MetaItem { path: _, node, span } = mi; match node { MetaItemKind::Word => {} MetaItemKind::List(mis) => visit_vec(mis, |mi| vis.visit_meta_list_item(mi)), @@ -557,17 +561,18 @@ pub fn noop_visit_meta_item<T: MutVisitor>(mi: &mut MetaItem, vis: &mut T) { vis.visit_span(span); } -pub fn noop_visit_arg<T: MutVisitor>(Arg { id, pat, ty }: &mut Arg, vis: &mut T) { +pub fn noop_visit_arg<T: MutVisitor>(Arg { attrs, id, pat, span, ty }: &mut Arg, vis: &mut T) { vis.visit_id(id); + visit_thin_attrs(attrs, vis); vis.visit_pat(pat); + vis.visit_span(span); vis.visit_ty(ty); } pub fn noop_visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) { match tt { - TokenTree::Token(span, tok) => { - vis.visit_span(span); - vis.visit_token(tok); + TokenTree::Token(token) => { + vis.visit_token(token); } TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { vis.visit_span(open); @@ -584,17 +589,26 @@ pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &m }) } -// apply ident visitor if it's an ident, apply other visits to interpolated nodes +// Apply ident visitor if it's an ident, apply other visits to interpolated nodes. +// In practice the ident part is not actually used by specific visitors right now, +// but there's a test below checking that it works. pub fn noop_visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) { - match t { - token::Ident(id, _is_raw) => vis.visit_ident(id), - token::Lifetime(id) => vis.visit_ident(id), + let Token { kind, span } = t; + match kind { + token::Ident(name, _) | token::Lifetime(name) => { + let mut ident = Ident::new(*name, *span); + vis.visit_ident(&mut ident); + *name = ident.name; + *span = ident.span; + return; // avoid visiting the span for the second time + } token::Interpolated(nt) => { let mut nt = Lrc::make_mut(nt); vis.visit_interpolated(&mut nt); } _ => {} } + vis.visit_span(span); } /// Apply visitor to elements of interpolated nodes. @@ -644,7 +658,6 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: token::NtMeta(meta) => vis.visit_meta_item(meta), token::NtPath(path) => vis.visit_path(path), token::NtTT(tt) => vis.visit_tt(tt), - token::NtArm(arm) => vis.visit_arm(arm), token::NtImplItem(item) => visit_clobber(item, |item| { // See reasoning above. @@ -657,9 +670,6 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: vis.flat_map_trait_item(item) .expect_one("expected visitor to produce exactly one item") }), - token::NtGenerics(generics) => vis.visit_generics(generics), - token::NtWhereClause(where_clause) => vis.visit_where_clause(where_clause), - token::NtArg(arg) => vis.visit_arg(arg), token::NtVis(visib) => vis.visit_vis(visib), token::NtForeignItem(item) => visit_clobber(item, |item| { @@ -734,8 +744,7 @@ pub fn noop_visit_generics<T: MutVisitor>(generics: &mut Generics, vis: &mut T) } pub fn noop_visit_where_clause<T: MutVisitor>(wc: &mut WhereClause, vis: &mut T) { - let WhereClause { id, predicates, span } = wc; - vis.visit_id(id); + let WhereClause { predicates, span } = wc; visit_vec(predicates, |predicate| vis.visit_where_predicate(predicate)); vis.visit_span(span); } @@ -767,11 +776,11 @@ pub fn noop_visit_where_predicate<T: MutVisitor>(pred: &mut WherePredicate, vis: pub fn noop_visit_variant_data<T: MutVisitor>(vdata: &mut VariantData, vis: &mut T) { match vdata { - VariantData::Struct(fields, id) | + VariantData::Struct(fields, ..) => visit_vec(fields, |field| vis.visit_struct_field(field)), VariantData::Tuple(fields, id) => { visit_vec(fields, |field| vis.visit_struct_field(field)); vis.visit_id(id); - } + }, VariantData::Unit(id) => vis.visit_id(id), } } @@ -838,11 +847,11 @@ pub fn noop_visit_item_kind<T: MutVisitor>(kind: &mut ItemKind, vis: &mut T) { ItemKind::Mod(m) => vis.visit_mod(m), ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm), ItemKind::GlobalAsm(_ga) => {} - ItemKind::Ty(ty, generics) => { + ItemKind::TyAlias(ty, generics) => { vis.visit_ty(ty); vis.visit_generics(generics); } - ItemKind::Existential(bounds, generics) => { + ItemKind::OpaqueTy(bounds, generics) => { visit_bounds(bounds, vis); vis.visit_generics(generics); } @@ -924,8 +933,8 @@ pub fn noop_flat_map_impl_item<T: MutVisitor>(mut item: ImplItem, visitor: &mut visit_method_sig(sig, visitor); visitor.visit_block(body); } - ImplItemKind::Type(ty) => visitor.visit_ty(ty), - ImplItemKind::Existential(bounds) => visit_bounds(bounds, visitor), + ImplItemKind::TyAlias(ty) => visitor.visit_ty(ty), + ImplItemKind::OpaqueTy(bounds) => visit_bounds(bounds, visitor), ImplItemKind::Macro(mac) => visitor.visit_mac(mac), } visitor.visit_span(span); @@ -946,7 +955,7 @@ pub fn noop_visit_mod<T: MutVisitor>(Mod { inner, items, inline: _ }: &mut Mod, pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) { visit_clobber(krate, |Crate { module, attrs, span }| { let item = P(Item { - ident: keywords::Invalid.ident(), + ident: Ident::invalid(), attrs, id: DUMMY_NODE_ID, vis: respan(span.shrink_to_lo(), VisibilityKind::Public), @@ -1015,15 +1024,15 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) { let Pat { id, node, span } = pat.deref_mut(); vis.visit_id(id); match node { - PatKind::Wild => {} + PatKind::Wild | PatKind::Rest => {} PatKind::Ident(_binding_mode, ident, sub) => { vis.visit_ident(ident); visit_opt(sub, |sub| vis.visit_pat(sub)); } PatKind::Lit(e) => vis.visit_expr(e), - PatKind::TupleStruct(path, pats, _ddpos) => { + PatKind::TupleStruct(path, elems) => { vis.visit_path(path); - visit_vec(pats, |pat| vis.visit_pat(pat)); + visit_vec(elems, |elem| vis.visit_pat(elem)); } PatKind::Path(qself, path) => { vis.visit_qself(qself); @@ -1038,7 +1047,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) { vis.visit_span(span); }; } - PatKind::Tuple(elts, _ddpos) => visit_vec(elts, |elt| vis.visit_pat(elt)), + PatKind::Tuple(elems) => visit_vec(elems, |elem| vis.visit_pat(elem)), PatKind::Box(inner) => vis.visit_pat(inner), PatKind::Ref(inner, _mutbl) => vis.visit_pat(inner), PatKind::Range(e1, e2, Spanned { span: _, node: _ }) => { @@ -1046,11 +1055,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) { vis.visit_expr(e2); vis.visit_span(span); } - PatKind::Slice(before, slice, after) => { - visit_vec(before, |pat| vis.visit_pat(pat)); - visit_opt(slice, |slice| vis.visit_pat(slice)); - visit_vec(after, |pat| vis.visit_pat(pat)); - } + PatKind::Slice(elems) => visit_vec(elems, |elem| vis.visit_pat(elem)), PatKind::Paren(inner) => vis.visit_pat(inner), PatKind::Mac(mac) => vis.visit_mac(mac), } @@ -1065,10 +1070,6 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo pub fn noop_visit_expr<T: MutVisitor>(Expr { node, id, span, attrs }: &mut Expr, vis: &mut T) { match node { ExprKind::Box(expr) => vis.visit_expr(expr), - ExprKind::ObsoleteInPlace(a, b) => { - vis.visit_expr(a); - vis.visit_expr(b); - } ExprKind::Array(exprs) => visit_exprs(exprs, vis), ExprKind::Repeat(expr, count) => { vis.visit_expr(expr); @@ -1090,7 +1091,6 @@ pub fn noop_visit_expr<T: MutVisitor>(Expr { node, id, span, attrs }: &mut Expr, vis.visit_expr(rhs); } ExprKind::Unary(_unop, ohs) => vis.visit_expr(ohs), - ExprKind::Lit(_lit) => {} ExprKind::Cast(expr, ty) => { vis.visit_expr(expr); vis.visit_ty(ty); @@ -1100,28 +1100,20 @@ pub fn noop_visit_expr<T: MutVisitor>(Expr { node, id, span, attrs }: &mut Expr, vis.visit_ty(ty); } ExprKind::AddrOf(_m, ohs) => vis.visit_expr(ohs), + ExprKind::Let(pats, scrutinee) => { + visit_vec(pats, |pat| vis.visit_pat(pat)); + vis.visit_expr(scrutinee); + } ExprKind::If(cond, tr, fl) => { vis.visit_expr(cond); vis.visit_block(tr); visit_opt(fl, |fl| vis.visit_expr(fl)); } - ExprKind::IfLet(pats, expr, tr, fl) => { - visit_vec(pats, |pat| vis.visit_pat(pat)); - vis.visit_expr(expr); - vis.visit_block(tr); - visit_opt(fl, |fl| vis.visit_expr(fl)); - } ExprKind::While(cond, body, label) => { vis.visit_expr(cond); vis.visit_block(body); visit_opt(label, |label| vis.visit_label(label)); } - ExprKind::WhileLet(pats, expr, body, label) => { - visit_vec(pats, |pat| vis.visit_pat(pat)); - vis.visit_expr(expr); - vis.visit_block(body); - visit_opt(label, |label| vis.visit_label(label)); - } ExprKind::ForLoop(pat, iter, body, label) => { vis.visit_pat(pat); vis.visit_expr(iter); @@ -1150,6 +1142,7 @@ pub fn noop_visit_expr<T: MutVisitor>(Expr { node, id, span, attrs }: &mut Expr, vis.visit_id(node_id); vis.visit_block(body); } + ExprKind::Await(expr) => vis.visit_expr(expr), ExprKind::Assign(el, er) => { vis.visit_expr(el); vis.visit_expr(er); @@ -1213,7 +1206,7 @@ pub fn noop_visit_expr<T: MutVisitor>(Expr { node, id, span, attrs }: &mut Expr, } ExprKind::Try(expr) => vis.visit_expr(expr), ExprKind::TryBlock(body) => vis.visit_block(body), - ExprKind::Err => {} + ExprKind::Lit(_) | ExprKind::Err => {} } vis.visit_id(id); vis.visit_span(span); @@ -1265,79 +1258,3 @@ pub fn noop_visit_vis<T: MutVisitor>(Spanned { node, span }: &mut Visibility, vi } vis.visit_span(span); } - -#[cfg(test)] -mod tests { - use std::io; - use crate::ast::{self, Ident}; - use crate::util::parser_testing::{string_to_crate, matches_codepattern}; - use crate::print::pprust; - use crate::mut_visit; - use crate::with_globals; - use super::*; - - // this version doesn't care about getting comments or docstrings in. - fn fake_print_crate(s: &mut pprust::State<'_>, - krate: &ast::Crate) -> io::Result<()> { - s.print_mod(&krate.module, &krate.attrs) - } - - // change every identifier to "zz" - struct ToZzIdentMutVisitor; - - impl MutVisitor for ToZzIdentMutVisitor { - fn visit_ident(&mut self, ident: &mut ast::Ident) { - *ident = Ident::from_str("zz"); - } - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - - // maybe add to expand.rs... - macro_rules! assert_pred { - ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( - { - let pred_val = $pred; - let a_val = $a; - let b_val = $b; - if !(pred_val(&a_val, &b_val)) { - panic!("expected args satisfying {}, got {} and {}", - $predname, a_val, b_val); - } - } - ) - } - - // make sure idents get transformed everywhere - #[test] fn ident_transformation () { - with_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); - }) - } - - // even inside macro defs.... - #[test] fn ident_transformation_in_defs () { - with_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ - (g $(d $d $e)+))} ".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "macro_rules! zz((zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+)));".to_string()); - }) - } -} - diff --git a/src/libsyntax/mut_visit/tests.rs b/src/libsyntax/mut_visit/tests.rs new file mode 100644 index 00000000000..6868736976b --- /dev/null +++ b/src/libsyntax/mut_visit/tests.rs @@ -0,0 +1,71 @@ +use super::*; + +use crate::ast::{self, Ident}; +use crate::tests::{string_to_crate, matches_codepattern}; +use crate::print::pprust; +use crate::mut_visit; +use crate::with_default_globals; + +// this version doesn't care about getting comments or docstrings in. +fn fake_print_crate(s: &mut pprust::State<'_>, + krate: &ast::Crate) { + s.print_mod(&krate.module, &krate.attrs) +} + +// change every identifier to "zz" +struct ToZzIdentMutVisitor; + +impl MutVisitor for ToZzIdentMutVisitor { + fn visit_ident(&mut self, ident: &mut ast::Ident) { + *ident = Ident::from_str("zz"); + } + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +// maybe add to expand.rs... +macro_rules! assert_pred { + ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( + { + let pred_val = $pred; + let a_val = $a; + let b_val = $b; + if !(pred_val(&a_val, &b_val)) { + panic!("expected args satisfying {}, got {} and {}", + $predname, a_val, b_val); + } + } + ) +} + +// make sure idents get transformed everywhere +#[test] fn ident_transformation () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); + }) +} + +// even inside macro defs.... +#[test] fn ident_transformation_in_defs () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ + (g $(d $d $e)+))} ".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string()); + }) +} diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index e93e15f9012..a42da112360 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -1,30 +1,39 @@ use crate::attr; use crate::ast; -use crate::source_map::respan; use crate::parse::{SeqSep, PResult}; use crate::parse::token::{self, Nonterminal, DelimToken}; use crate::parse::parser::{Parser, TokenType, PathStyle}; use crate::tokenstream::{TokenStream, TokenTree}; +use crate::source_map::Span; use log::debug; +use smallvec::smallvec; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { Permitted, - NotPermitted { reason: &'a str }, + NotPermitted { reason: &'a str, saw_doc_comment: bool, prev_attr_sp: Option<Span> }, } const DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG: &str = "an inner attribute is not \ permitted in this context"; impl<'a> Parser<'a> { + crate fn parse_arg_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { + let attrs = self.parse_outer_attributes()?; + attrs.iter().for_each(|a| + self.sess.param_attr_spans.borrow_mut().push(a.span) + ); + Ok(attrs) + } + /// Parse attributes that appear before an item crate fn parse_outer_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { let mut attrs: Vec<ast::Attribute> = Vec::new(); let mut just_parsed_doc_comment = false; loop { debug!("parse_outer_attributes: self.token={:?}", self.token); - match self.token { + match self.token.kind { token::Pound => { let inner_error_reason = if just_parsed_doc_comment { "an inner attribute is not permitted following an outer doc comment" @@ -34,12 +43,17 @@ impl<'a> Parser<'a> { DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG }; let inner_parse_policy = - InnerAttributeParsePolicy::NotPermitted { reason: inner_error_reason }; - attrs.push(self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?); + InnerAttributeParsePolicy::NotPermitted { + reason: inner_error_reason, + saw_doc_comment: just_parsed_doc_comment, + prev_attr_sp: attrs.last().and_then(|a| Some(a.span)) + }; + let attr = self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; + attrs.push(attr); just_parsed_doc_comment = false; } token::DocComment(s) => { - let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span); + let attr = attr::mk_sugared_doc_attr(s, self.token.span); if attr.style != ast::AttrStyle::Outer { let mut err = self.fatal("expected outer doc comment"); err.note("inner doc comments like this (starting with \ @@ -67,8 +81,11 @@ impl<'a> Parser<'a> { let inner_parse_policy = if permit_inner { InnerAttributeParsePolicy::Permitted } else { - InnerAttributeParsePolicy::NotPermitted - { reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG } + InnerAttributeParsePolicy::NotPermitted { + reason: DEFAULT_UNEXPECTED_INNER_ATTR_ERR_MSG, + saw_doc_comment: false, + prev_attr_sp: None + } }; self.parse_attribute_with_inner_parse_policy(inner_parse_policy) } @@ -81,27 +98,17 @@ impl<'a> Parser<'a> { debug!("parse_attribute_with_inner_parse_policy: inner_parse_policy={:?} self.token={:?}", inner_parse_policy, self.token); - let (span, path, tokens, style) = match self.token { + let (span, path, tokens, style) = match self.token.kind { token::Pound => { - let lo = self.span; + let lo = self.token.span; self.bump(); if let InnerAttributeParsePolicy::Permitted = inner_parse_policy { self.expected_tokens.push(TokenType::Token(token::Not)); } + let style = if self.token == token::Not { self.bump(); - if let InnerAttributeParsePolicy::NotPermitted { reason } = inner_parse_policy - { - let span = self.span; - self.diagnostic() - .struct_span_err(span, reason) - .note("inner attributes, like `#![no_std]`, annotate the item \ - enclosing them, and are usually found at the beginning of \ - source files. Outer attributes, like `#[test]`, annotate the \ - item following them.") - .emit() - } ast::AttrStyle::Inner } else { ast::AttrStyle::Outer @@ -112,7 +119,38 @@ impl<'a> Parser<'a> { self.expect(&token::CloseDelim(token::Bracket))?; let hi = self.prev_span; - (lo.to(hi), path, tokens, style) + let attr_sp = lo.to(hi); + + // Emit error if inner attribute is encountered and not permitted + if style == ast::AttrStyle::Inner { + if let InnerAttributeParsePolicy::NotPermitted { reason, + saw_doc_comment, prev_attr_sp } = inner_parse_policy { + let prev_attr_note = if saw_doc_comment { + "previous doc comment" + } else { + "previous outer attribute" + }; + + let mut diagnostic = self + .diagnostic() + .struct_span_err(attr_sp, reason); + + if let Some(prev_attr_sp) = prev_attr_sp { + diagnostic + .span_label(attr_sp, "not permitted following an outer attibute") + .span_label(prev_attr_sp, prev_attr_note); + } + + diagnostic + .note("inner attributes, like `#![no_std]`, annotate the item \ + enclosing them, and are usually found at the beginning of \ + source files. Outer attributes, like `#[test]`, annotate the \ + item following them.") + .emit() + } + } + + (attr_sp, path, tokens, style) } _ => { let token_str = self.this_token_to_string(); @@ -140,7 +178,7 @@ impl<'a> Parser<'a> { /// PATH `=` TOKEN_TREE /// The delimiters or `=` are still put into the resulting token stream. crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> { - let meta = match self.token { + let meta = match self.token.kind { token::Interpolated(ref nt) => match **nt { Nonterminal::NtMeta(ref meta) => Some(meta.clone()), _ => None, @@ -149,7 +187,7 @@ impl<'a> Parser<'a> { }; Ok(if let Some(meta) = meta { self.bump(); - (meta.ident, meta.node.tokens(meta.span)) + (meta.path, meta.node.tokens(meta.span)) } else { let path = self.parse_path(PathStyle::Mod)?; let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) || @@ -157,9 +195,9 @@ impl<'a> Parser<'a> { self.check(&token::OpenDelim(DelimToken::Brace)) { self.parse_token_tree().into() } else if self.eat(&token::Eq) { - let eq = TokenTree::Token(self.prev_span, token::Eq); + let eq = TokenTree::token(token::Eq, self.prev_span); let mut is_interpolated_expr = false; - if let token::Interpolated(nt) = &self.token { + if let token::Interpolated(nt) = &self.token.kind { if let token::NtExpr(..) = **nt { is_interpolated_expr = true; } @@ -172,7 +210,7 @@ impl<'a> Parser<'a> { } else { self.parse_unsuffixed_lit()?.tokens() }; - TokenStream::from_streams(vec![eq.into(), tokens]) + TokenStream::from_streams(smallvec![eq.into(), tokens]) } else { TokenStream::empty() }; @@ -188,7 +226,7 @@ impl<'a> Parser<'a> { crate fn parse_inner_attributes(&mut self) -> PResult<'a, Vec<ast::Attribute>> { let mut attrs: Vec<ast::Attribute> = vec![]; loop { - match self.token { + match self.token.kind { token::Pound => { // Don't even try to parse if it's not an inner attribute. if !self.look_ahead(1, |t| t == &token::Not) { @@ -201,7 +239,7 @@ impl<'a> Parser<'a> { } token::DocComment(s) => { // we need to get the position of this token before we bump. - let attr = attr::mk_sugared_doc_attr(attr::mk_attr_id(), s, self.span); + let attr = attr::mk_sugared_doc_attr(s, self.token.span); if attr.style == ast::AttrStyle::Inner { attrs.push(attr); self.bump(); @@ -217,7 +255,7 @@ impl<'a> Parser<'a> { fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> { let lit = self.parse_lit()?; - debug!("Checking if {:?} is unusuffixed.", lit); + debug!("checking if {:?} is unusuffixed", lit); if !lit.node.is_unsuffixed() { let msg = "suffixed literals are not allowed in attributes"; @@ -236,7 +274,7 @@ impl<'a> Parser<'a> { /// meta_item : IDENT ( '=' UNSUFFIXED_LIT | '(' meta_item_inner? ')' )? ; /// meta_item_inner : (meta_item | UNSUFFIXED_LIT) (',' meta_item_inner)? ; pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> { - let nt_meta = match self.token { + let nt_meta = match self.token.kind { token::Interpolated(ref nt) => match **nt { token::NtMeta(ref e) => Some(e.clone()), _ => None, @@ -249,11 +287,11 @@ impl<'a> Parser<'a> { return Ok(meta); } - let lo = self.span; - let ident = self.parse_path(PathStyle::Mod)?; + let lo = self.token.span; + let path = self.parse_path(PathStyle::Mod)?; let node = self.parse_meta_item_kind()?; let span = lo.to(self.prev_span); - Ok(ast::MetaItem { ident, node, span }) + Ok(ast::MetaItem { path, node, span }) } crate fn parse_meta_item_kind(&mut self) -> PResult<'a, ast::MetaItemKind> { @@ -268,25 +306,23 @@ impl<'a> Parser<'a> { /// matches meta_item_inner : (meta_item | UNSUFFIXED_LIT) ; fn parse_meta_item_inner(&mut self) -> PResult<'a, ast::NestedMetaItem> { - let lo = self.span; - match self.parse_unsuffixed_lit() { Ok(lit) => { - return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::Literal(lit))) + return Ok(ast::NestedMetaItem::Literal(lit)) } Err(ref mut err) => self.diagnostic().cancel(err) } match self.parse_meta_item() { Ok(mi) => { - return Ok(respan(lo.to(self.prev_span), ast::NestedMetaItemKind::MetaItem(mi))) + return Ok(ast::NestedMetaItem::MetaItem(mi)) } Err(ref mut err) => self.diagnostic().cancel(err) } let found = self.this_token_to_string(); let msg = format!("expected unsuffixed literal or identifier, found `{}`", found); - Err(self.diagnostic().struct_span_err(self.span, &msg)) + Err(self.diagnostic().struct_span_err(self.token.span, &msg)) } /// matches meta_seq = ( COMMASEP(meta_item_inner) ) diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs index b4103440e35..6ebfab3a133 100644 --- a/src/libsyntax/parse/classify.rs +++ b/src/libsyntax/parse/classify.rs @@ -14,27 +14,12 @@ use crate::ast; pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool { match e.node { ast::ExprKind::If(..) | - ast::ExprKind::IfLet(..) | ast::ExprKind::Match(..) | ast::ExprKind::Block(..) | ast::ExprKind::While(..) | - ast::ExprKind::WhileLet(..) | ast::ExprKind::Loop(..) | ast::ExprKind::ForLoop(..) | ast::ExprKind::TryBlock(..) => false, _ => true, } } - -/// this statement requires a semicolon after it. -/// note that in one case (`stmt_semi`), we've already -/// seen the semicolon, and thus don't need another. -pub fn stmt_ends_with_semi(stmt: &ast::StmtKind) -> bool { - match *stmt { - ast::StmtKind::Local(_) => true, - ast::StmtKind::Expr(ref e) => expr_requires_semi_to_be_stmt(e), - ast::StmtKind::Item(_) | - ast::StmtKind::Semi(..) | - ast::StmtKind::Mac(..) => false, - } -} diff --git a/src/libsyntax/parse/diagnostics.rs b/src/libsyntax/parse/diagnostics.rs new file mode 100644 index 00000000000..730efb5ef01 --- /dev/null +++ b/src/libsyntax/parse/diagnostics.rs @@ -0,0 +1,1374 @@ +use crate::ast::{ + self, Arg, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, + Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, VariantData, +}; +use crate::feature_gate::{feature_err, UnstableFeatures}; +use crate::parse::{SeqSep, PResult, Parser, ParseSess}; +use crate::parse::parser::{BlockMode, PathStyle, SemiColonMode, TokenType, TokenExpectType}; +use crate::parse::token::{self, TokenKind}; +use crate::print::pprust; +use crate::ptr::P; +use crate::source_map::Spanned; +use crate::symbol::{kw, sym}; +use crate::ThinVec; +use crate::util::parser::AssocOp; +use errors::{Applicability, DiagnosticBuilder, DiagnosticId}; +use rustc_data_structures::fx::FxHashSet; +use syntax_pos::{Span, DUMMY_SP, MultiSpan, SpanSnippetError}; +use log::{debug, trace}; +use std::mem; + +/// Creates a placeholder argument. +crate fn dummy_arg(ident: Ident) -> Arg { + let pat = P(Pat { + id: ast::DUMMY_NODE_ID, + node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None), + span: ident.span, + }); + let ty = Ty { + node: TyKind::Err, + span: ident.span, + id: ast::DUMMY_NODE_ID + }; + Arg { attrs: ThinVec::default(), id: ast::DUMMY_NODE_ID, pat, span: ident.span, ty: P(ty) } +} + +pub enum Error { + FileNotFoundForModule { + mod_name: String, + default_path: String, + secondary_path: String, + dir_path: String, + }, + DuplicatePaths { + mod_name: String, + default_path: String, + secondary_path: String, + }, + UselessDocComment, + InclusiveRangeWithNoEnd, +} + +impl Error { + fn span_err<S: Into<MultiSpan>>( + self, + sp: S, + handler: &errors::Handler, + ) -> DiagnosticBuilder<'_> { + match self { + Error::FileNotFoundForModule { + ref mod_name, + ref default_path, + ref secondary_path, + ref dir_path, + } => { + let mut err = struct_span_err!( + handler, + sp, + E0583, + "file not found for module `{}`", + mod_name, + ); + err.help(&format!( + "name the file either {} or {} inside the directory \"{}\"", + default_path, + secondary_path, + dir_path, + )); + err + } + Error::DuplicatePaths { ref mod_name, ref default_path, ref secondary_path } => { + let mut err = struct_span_err!( + handler, + sp, + E0584, + "file for module `{}` found at both {} and {}", + mod_name, + default_path, + secondary_path, + ); + err.help("delete or rename one of them to remove the ambiguity"); + err + } + Error::UselessDocComment => { + let mut err = struct_span_err!( + handler, + sp, + E0585, + "found a documentation comment that doesn't document anything", + ); + err.help("doc comments must come before what they document, maybe a comment was \ + intended with `//`?"); + err + } + Error::InclusiveRangeWithNoEnd => { + let mut err = struct_span_err!( + handler, + sp, + E0586, + "inclusive range with no end", + ); + err.help("inclusive ranges must be bounded at the end (`..=b` or `a..=b`)"); + err + } + } + } +} + +pub trait RecoverQPath: Sized + 'static { + const PATH_STYLE: PathStyle = PathStyle::Expr; + fn to_ty(&self) -> Option<P<Ty>>; + fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self; +} + +impl RecoverQPath for Ty { + const PATH_STYLE: PathStyle = PathStyle::Type; + fn to_ty(&self) -> Option<P<Ty>> { + Some(P(self.clone())) + } + fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self { + Self { + span: path.span, + node: TyKind::Path(qself, path), + id: ast::DUMMY_NODE_ID, + } + } +} + +impl RecoverQPath for Pat { + fn to_ty(&self) -> Option<P<Ty>> { + self.to_ty() + } + fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self { + Self { + span: path.span, + node: PatKind::Path(qself, path), + id: ast::DUMMY_NODE_ID, + } + } +} + +impl RecoverQPath for Expr { + fn to_ty(&self) -> Option<P<Ty>> { + self.to_ty() + } + fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self { + Self { + span: path.span, + node: ExprKind::Path(qself, path), + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + } + } +} + +impl<'a> Parser<'a> { + pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { + self.span_fatal(self.token.span, m) + } + + pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + self.sess.span_diagnostic.struct_span_fatal(sp, m) + } + + pub fn span_fatal_err<S: Into<MultiSpan>>(&self, sp: S, err: Error) -> DiagnosticBuilder<'a> { + err.span_err(sp, self.diagnostic()) + } + + pub fn bug(&self, m: &str) -> ! { + self.sess.span_diagnostic.span_bug(self.token.span, m) + } + + pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) { + self.sess.span_diagnostic.span_err(sp, m) + } + + crate fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { + self.sess.span_diagnostic.struct_span_err(sp, m) + } + + crate fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! { + self.sess.span_diagnostic.span_bug(sp, m) + } + + crate fn cancel(&self, err: &mut DiagnosticBuilder<'_>) { + self.sess.span_diagnostic.cancel(err) + } + + crate fn diagnostic(&self) -> &'a errors::Handler { + &self.sess.span_diagnostic + } + + crate fn span_to_snippet(&self, span: Span) -> Result<String, SpanSnippetError> { + self.sess.source_map().span_to_snippet(span) + } + + crate fn expected_ident_found(&self) -> DiagnosticBuilder<'a> { + let mut err = self.struct_span_err( + self.token.span, + &format!("expected identifier, found {}", self.this_token_descr()), + ); + if let token::Ident(name, false) = self.token.kind { + if Ident::new(name, self.token.span).is_raw_guess() { + err.span_suggestion( + self.token.span, + "you can escape reserved keywords to use them as identifiers", + format!("r#{}", name), + Applicability::MaybeIncorrect, + ); + } + } + if let Some(token_descr) = self.token_descr() { + err.span_label(self.token.span, format!("expected identifier, found {}", token_descr)); + } else { + err.span_label(self.token.span, "expected identifier"); + if self.token == token::Comma && self.look_ahead(1, |t| t.is_ident()) { + err.span_suggestion( + self.token.span, + "remove this comma", + String::new(), + Applicability::MachineApplicable, + ); + } + } + err + } + + pub fn expected_one_of_not_found( + &mut self, + edible: &[TokenKind], + inedible: &[TokenKind], + ) -> PResult<'a, bool /* recovered */> { + fn tokens_to_string(tokens: &[TokenType]) -> String { + let mut i = tokens.iter(); + // This might be a sign we need a connect method on Iterator. + let b = i.next() + .map_or(String::new(), |t| t.to_string()); + i.enumerate().fold(b, |mut b, (i, a)| { + if tokens.len() > 2 && i == tokens.len() - 2 { + b.push_str(", or "); + } else if tokens.len() == 2 && i == tokens.len() - 2 { + b.push_str(" or "); + } else { + b.push_str(", "); + } + b.push_str(&a.to_string()); + b + }) + } + + let mut expected = edible.iter() + .map(|x| TokenType::Token(x.clone())) + .chain(inedible.iter().map(|x| TokenType::Token(x.clone()))) + .chain(self.expected_tokens.iter().cloned()) + .collect::<Vec<_>>(); + expected.sort_by_cached_key(|x| x.to_string()); + expected.dedup(); + let expect = tokens_to_string(&expected[..]); + let actual = self.this_token_to_string(); + let (msg_exp, (label_sp, label_exp)) = if expected.len() > 1 { + let short_expect = if expected.len() > 6 { + format!("{} possible tokens", expected.len()) + } else { + expect.clone() + }; + (format!("expected one of {}, found `{}`", expect, actual), + (self.sess.source_map().next_point(self.prev_span), + format!("expected one of {} here", short_expect))) + } else if expected.is_empty() { + (format!("unexpected token: `{}`", actual), + (self.prev_span, "unexpected token after this".to_string())) + } else { + (format!("expected {}, found `{}`", expect, actual), + (self.sess.source_map().next_point(self.prev_span), + format!("expected {} here", expect))) + }; + self.last_unexpected_token_span = Some(self.token.span); + let mut err = self.fatal(&msg_exp); + if self.token.is_ident_named(sym::and) { + err.span_suggestion_short( + self.token.span, + "use `&&` instead of `and` for the boolean operator", + "&&".to_string(), + Applicability::MaybeIncorrect, + ); + } + if self.token.is_ident_named(sym::or) { + err.span_suggestion_short( + self.token.span, + "use `||` instead of `or` for the boolean operator", + "||".to_string(), + Applicability::MaybeIncorrect, + ); + } + let sp = if self.token == token::Eof { + // This is EOF, don't want to point at the following char, but rather the last token + self.prev_span + } else { + label_sp + }; + match self.recover_closing_delimiter(&expected.iter().filter_map(|tt| match tt { + TokenType::Token(t) => Some(t.clone()), + _ => None, + }).collect::<Vec<_>>(), err) { + Err(e) => err = e, + Ok(recovered) => { + return Ok(recovered); + } + } + + let is_semi_suggestable = expected.iter().any(|t| match t { + TokenType::Token(token::Semi) => true, // we expect a `;` here + _ => false, + }) && ( // a `;` would be expected before the current keyword + self.token.is_keyword(kw::Break) || + self.token.is_keyword(kw::Continue) || + self.token.is_keyword(kw::For) || + self.token.is_keyword(kw::If) || + self.token.is_keyword(kw::Let) || + self.token.is_keyword(kw::Loop) || + self.token.is_keyword(kw::Match) || + self.token.is_keyword(kw::Return) || + self.token.is_keyword(kw::While) + ); + let sm = self.sess.source_map(); + match (sm.lookup_line(self.token.span.lo()), sm.lookup_line(sp.lo())) { + (Ok(ref a), Ok(ref b)) if a.line != b.line && is_semi_suggestable => { + // The spans are in different lines, expected `;` and found `let` or `return`. + // High likelihood that it is only a missing `;`. + err.span_suggestion_short( + label_sp, + "a semicolon may be missing here", + ";".to_string(), + Applicability::MaybeIncorrect, + ); + err.emit(); + return Ok(true); + } + (Ok(ref a), Ok(ref b)) if a.line == b.line => { + // When the spans are in the same line, it means that the only content between + // them is whitespace, point at the found token in that case: + // + // X | () => { syntax error }; + // | ^^^^^ expected one of 8 possible tokens here + // + // instead of having: + // + // X | () => { syntax error }; + // | -^^^^^ unexpected token + // | | + // | expected one of 8 possible tokens here + err.span_label(self.token.span, label_exp); + } + _ if self.prev_span == syntax_pos::DUMMY_SP => { + // Account for macro context where the previous span might not be + // available to avoid incorrect output (#54841). + err.span_label(self.token.span, "unexpected token"); + } + _ => { + err.span_label(sp, label_exp); + err.span_label(self.token.span, "unexpected token"); + } + } + self.maybe_annotate_with_ascription(&mut err, false); + Err(err) + } + + pub fn maybe_annotate_with_ascription( + &self, + err: &mut DiagnosticBuilder<'_>, + maybe_expected_semicolon: bool, + ) { + if let Some((sp, likely_path)) = self.last_type_ascription { + let sm = self.sess.source_map(); + let next_pos = sm.lookup_char_pos(self.token.span.lo()); + let op_pos = sm.lookup_char_pos(sp.hi()); + + if likely_path { + err.span_suggestion( + sp, + "maybe write a path separator here", + "::".to_string(), + match self.sess.unstable_features { + UnstableFeatures::Disallow => Applicability::MachineApplicable, + _ => Applicability::MaybeIncorrect, + }, + ); + } else if op_pos.line != next_pos.line && maybe_expected_semicolon { + err.span_suggestion( + sp, + "try using a semicolon", + ";".to_string(), + Applicability::MaybeIncorrect, + ); + } else if let UnstableFeatures::Disallow = self.sess.unstable_features { + err.span_label(sp, "tried to parse a type due to this"); + } else { + err.span_label(sp, "tried to parse a type due to this type ascription"); + } + if let UnstableFeatures::Disallow = self.sess.unstable_features { + // Give extra information about type ascription only if it's a nightly compiler. + } else { + err.note("`#![feature(type_ascription)]` lets you annotate an expression with a \ + type: `<expr>: <type>`"); + err.note("for more information, see \ + https://github.com/rust-lang/rust/issues/23416"); + } + } + } + + /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, + /// passes through any errors encountered. Used for error recovery. + crate fn eat_to_tokens(&mut self, kets: &[&TokenKind]) { + let handler = self.diagnostic(); + + if let Err(ref mut err) = self.parse_seq_to_before_tokens( + kets, + SeqSep::none(), + TokenExpectType::Expect, + |p| Ok(p.parse_token_tree()), + ) { + handler.cancel(err); + } + } + + /// This function checks if there are trailing angle brackets and produces + /// a diagnostic to suggest removing them. + /// + /// ```ignore (diagnostic) + /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>(); + /// ^^ help: remove extra angle brackets + /// ``` + crate fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: TokenKind) { + // This function is intended to be invoked after parsing a path segment where there are two + // cases: + // + // 1. A specific token is expected after the path segment. + // eg. `x.foo(`, `x.foo::<u32>(` (parenthesis - method call), + // `Foo::`, or `Foo::<Bar>::` (mod sep - continued path). + // 2. No specific token is expected after the path segment. + // eg. `x.foo` (field access) + // + // This function is called after parsing `.foo` and before parsing the token `end` (if + // present). This includes any angle bracket arguments, such as `.foo::<u32>` or + // `Foo::<Bar>`. + + // We only care about trailing angle brackets if we previously parsed angle bracket + // arguments. This helps stop us incorrectly suggesting that extra angle brackets be + // removed in this case: + // + // `x.foo >> (3)` (where `x.foo` is a `u32` for example) + // + // This case is particularly tricky as we won't notice it just looking at the tokens - + // it will appear the same (in terms of upcoming tokens) as below (since the `::<u32>` will + // have already been parsed): + // + // `x.foo::<u32>>>(3)` + let parsed_angle_bracket_args = segment.args + .as_ref() + .map(|args| args.is_angle_bracketed()) + .unwrap_or(false); + + debug!( + "check_trailing_angle_brackets: parsed_angle_bracket_args={:?}", + parsed_angle_bracket_args, + ); + if !parsed_angle_bracket_args { + return; + } + + // Keep the span at the start so we can highlight the sequence of `>` characters to be + // removed. + let lo = self.token.span; + + // We need to look-ahead to see if we have `>` characters without moving the cursor forward + // (since we might have the field access case and the characters we're eating are + // actual operators and not trailing characters - ie `x.foo >> 3`). + let mut position = 0; + + // We can encounter `>` or `>>` tokens in any order, so we need to keep track of how + // many of each (so we can correctly pluralize our error messages) and continue to + // advance. + let mut number_of_shr = 0; + let mut number_of_gt = 0; + while self.look_ahead(position, |t| { + trace!("check_trailing_angle_brackets: t={:?}", t); + if *t == token::BinOp(token::BinOpToken::Shr) { + number_of_shr += 1; + true + } else if *t == token::Gt { + number_of_gt += 1; + true + } else { + false + } + }) { + position += 1; + } + + // If we didn't find any trailing `>` characters, then we have nothing to error about. + debug!( + "check_trailing_angle_brackets: number_of_gt={:?} number_of_shr={:?}", + number_of_gt, number_of_shr, + ); + if number_of_gt < 1 && number_of_shr < 1 { + return; + } + + // Finally, double check that we have our end token as otherwise this is the + // second case. + if self.look_ahead(position, |t| { + trace!("check_trailing_angle_brackets: t={:?}", t); + *t == end + }) { + // Eat from where we started until the end token so that parsing can continue + // as if we didn't have those extra angle brackets. + self.eat_to_tokens(&[&end]); + let span = lo.until(self.token.span); + + let plural = number_of_gt > 1 || number_of_shr >= 1; + self.diagnostic() + .struct_span_err( + span, + &format!("unmatched angle bracket{}", if plural { "s" } else { "" }), + ) + .span_suggestion( + span, + &format!("remove extra angle bracket{}", if plural { "s" } else { "" }), + String::new(), + Applicability::MachineApplicable, + ) + .emit(); + } + } + + /// Produce an error if comparison operators are chained (RFC #558). + /// We only need to check lhs, not rhs, because all comparison ops + /// have same precedence and are left-associative + crate fn check_no_chained_comparison(&self, lhs: &Expr, outer_op: &AssocOp) { + debug_assert!(outer_op.is_comparison(), + "check_no_chained_comparison: {:?} is not comparison", + outer_op); + match lhs.node { + ExprKind::Binary(op, _, _) if op.node.is_comparison() => { + // respan to include both operators + let op_span = op.span.to(self.token.span); + let mut err = self.struct_span_err( + op_span, + "chained comparison operators require parentheses", + ); + if op.node == BinOpKind::Lt && + *outer_op == AssocOp::Less || // Include `<` to provide this recommendation + *outer_op == AssocOp::Greater // even in a case like the following: + { // Foo<Bar<Baz<Qux, ()>>> + err.help( + "use `::<...>` instead of `<...>` if you meant to specify type arguments"); + err.help("or use `(...)` if you meant to specify fn arguments"); + } + err.emit(); + } + _ => {} + } + } + + crate fn maybe_report_ambiguous_plus( + &mut self, + allow_plus: bool, + impl_dyn_multi: bool, + ty: &Ty, + ) { + if !allow_plus && impl_dyn_multi { + let sum_with_parens = format!("({})", pprust::ty_to_string(&ty)); + self.struct_span_err(ty.span, "ambiguous `+` in a type") + .span_suggestion( + ty.span, + "use parentheses to disambiguate", + sum_with_parens, + Applicability::MachineApplicable, + ) + .emit(); + } + } + + crate fn maybe_report_invalid_custom_discriminants( + sess: &ParseSess, + variants: &[Spanned<ast::Variant_>], + ) { + let has_fields = variants.iter().any(|variant| match variant.node.data { + VariantData::Tuple(..) | VariantData::Struct(..) => true, + VariantData::Unit(..) => false, + }); + + let discriminant_spans = variants.iter().filter(|variant| match variant.node.data { + VariantData::Tuple(..) | VariantData::Struct(..) => false, + VariantData::Unit(..) => true, + }) + .filter_map(|variant| variant.node.disr_expr.as_ref().map(|c| c.value.span)) + .collect::<Vec<_>>(); + + if !discriminant_spans.is_empty() && has_fields { + let mut err = feature_err( + sess, + sym::arbitrary_enum_discriminant, + discriminant_spans.clone(), + crate::feature_gate::GateIssue::Language, + "custom discriminant values are not allowed in enums with tuple or struct variants", + ); + for sp in discriminant_spans { + err.span_label(sp, "disallowed custom discriminant"); + } + for variant in variants.iter() { + match &variant.node.data { + VariantData::Struct(..) => { + err.span_label( + variant.span, + "struct variant defined here", + ); + } + VariantData::Tuple(..) => { + err.span_label( + variant.span, + "tuple variant defined here", + ); + } + VariantData::Unit(..) => {} + } + } + err.emit(); + } + } + + crate fn maybe_recover_from_bad_type_plus( + &mut self, + allow_plus: bool, + ty: &Ty, + ) -> PResult<'a, ()> { + // Do not add `+` to expected tokens. + if !allow_plus || !self.token.is_like_plus() { + return Ok(()); + } + + self.bump(); // `+` + let bounds = self.parse_generic_bounds(None)?; + let sum_span = ty.span.to(self.prev_span); + + let mut err = struct_span_err!( + self.sess.span_diagnostic, + sum_span, + E0178, + "expected a path on the left-hand side of `+`, not `{}`", + pprust::ty_to_string(ty) + ); + + match ty.node { + TyKind::Rptr(ref lifetime, ref mut_ty) => { + let sum_with_parens = pprust::to_string(|s| { + s.s.word("&"); + s.print_opt_lifetime(lifetime); + s.print_mutability(mut_ty.mutbl); + s.popen(); + s.print_type(&mut_ty.ty); + s.print_type_bounds(" +", &bounds); + s.pclose() + }); + err.span_suggestion( + sum_span, + "try adding parentheses", + sum_with_parens, + Applicability::MachineApplicable, + ); + } + TyKind::Ptr(..) | TyKind::BareFn(..) => { + err.span_label(sum_span, "perhaps you forgot parentheses?"); + } + _ => { + err.span_label(sum_span, "expected a path"); + } + } + err.emit(); + Ok(()) + } + + /// Try to recover from associated item paths like `[T]::AssocItem`/`(T, U)::AssocItem`. + /// Attempt to convert the base expression/pattern/type into a type, parse the `::AssocItem` + /// tail, and combine them into a `<Ty>::AssocItem` expression/pattern/type. + crate fn maybe_recover_from_bad_qpath<T: RecoverQPath>( + &mut self, + base: P<T>, + allow_recovery: bool, + ) -> PResult<'a, P<T>> { + // Do not add `::` to expected tokens. + if allow_recovery && self.token == token::ModSep { + if let Some(ty) = base.to_ty() { + return self.maybe_recover_from_bad_qpath_stage_2(ty.span, ty); + } + } + Ok(base) + } + + /// Given an already parsed `Ty` parse the `::AssocItem` tail and + /// combine them into a `<Ty>::AssocItem` expression/pattern/type. + crate fn maybe_recover_from_bad_qpath_stage_2<T: RecoverQPath>( + &mut self, + ty_span: Span, + ty: P<Ty>, + ) -> PResult<'a, P<T>> { + self.expect(&token::ModSep)?; + + let mut path = ast::Path { + segments: Vec::new(), + span: DUMMY_SP, + }; + self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?; + path.span = ty_span.to(self.prev_span); + + let ty_str = self + .span_to_snippet(ty_span) + .unwrap_or_else(|_| pprust::ty_to_string(&ty)); + self.diagnostic() + .struct_span_err(path.span, "missing angle brackets in associated item path") + .span_suggestion( + // this is a best-effort recovery + path.span, + "try", + format!("<{}>::{}", ty_str, path), + Applicability::MaybeIncorrect, + ) + .emit(); + + let path_span = ty_span.shrink_to_hi(); // use an empty path since `position` == 0 + Ok(P(T::recovered( + Some(QSelf { + ty, + path_span, + position: 0, + }), + path, + ))) + } + + crate fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool { + if self.eat(&token::Semi) { + let mut err = self.struct_span_err(self.prev_span, "expected item, found `;`"); + err.span_suggestion_short( + self.prev_span, + "remove this semicolon", + String::new(), + Applicability::MachineApplicable, + ); + if !items.is_empty() { + let previous_item = &items[items.len() - 1]; + let previous_item_kind_name = match previous_item.node { + // say "braced struct" because tuple-structs and + // braceless-empty-struct declarations do take a semicolon + ItemKind::Struct(..) => Some("braced struct"), + ItemKind::Enum(..) => Some("enum"), + ItemKind::Trait(..) => Some("trait"), + ItemKind::Union(..) => Some("union"), + _ => None, + }; + if let Some(name) = previous_item_kind_name { + err.help(&format!( + "{} declarations are not followed by a semicolon", + name + )); + } + } + err.emit(); + true + } else { + false + } + } + + /// Create a `DiagnosticBuilder` for an unexpected token `t` and try to recover if it is a + /// closing delimiter. + pub fn unexpected_try_recover( + &mut self, + t: &TokenKind, + ) -> PResult<'a, bool /* recovered */> { + let token_str = pprust::token_kind_to_string(t); + let this_token_str = self.this_token_descr(); + let (prev_sp, sp) = match (&self.token.kind, self.subparser_name) { + // Point at the end of the macro call when reaching end of macro arguments. + (token::Eof, Some(_)) => { + let sp = self.sess.source_map().next_point(self.token.span); + (sp, sp) + } + // We don't want to point at the following span after DUMMY_SP. + // This happens when the parser finds an empty TokenStream. + _ if self.prev_span == DUMMY_SP => (self.token.span, self.token.span), + // EOF, don't want to point at the following char, but rather the last token. + (token::Eof, None) => (self.prev_span, self.token.span), + _ => (self.sess.source_map().next_point(self.prev_span), self.token.span), + }; + let msg = format!( + "expected `{}`, found {}", + token_str, + match (&self.token.kind, self.subparser_name) { + (token::Eof, Some(origin)) => format!("end of {}", origin), + _ => this_token_str, + }, + ); + let mut err = self.struct_span_err(sp, &msg); + let label_exp = format!("expected `{}`", token_str); + match self.recover_closing_delimiter(&[t.clone()], err) { + Err(e) => err = e, + Ok(recovered) => { + return Ok(recovered); + } + } + let sm = self.sess.source_map(); + match (sm.lookup_line(prev_sp.lo()), sm.lookup_line(sp.lo())) { + (Ok(ref a), Ok(ref b)) if a.line == b.line => { + // When the spans are in the same line, it means that the only content + // between them is whitespace, point only at the found token. + err.span_label(sp, label_exp); + } + _ => { + err.span_label(prev_sp, label_exp); + err.span_label(sp, "unexpected token"); + } + } + Err(err) + } + + crate fn parse_semi_or_incorrect_foreign_fn_body( + &mut self, + ident: &Ident, + extern_sp: Span, + ) -> PResult<'a, ()> { + if self.token != token::Semi { + // this might be an incorrect fn definition (#62109) + let parser_snapshot = self.clone(); + match self.parse_inner_attrs_and_block() { + Ok((_, body)) => { + self.struct_span_err(ident.span, "incorrect `fn` inside `extern` block") + .span_label(ident.span, "can't have a body") + .span_label(body.span, "this body is invalid here") + .span_label( + extern_sp, + "`extern` blocks define existing foreign functions and `fn`s \ + inside of them cannot have a body") + .help("you might have meant to write a function accessible through ffi, \ + which can be done by writing `extern fn` outside of the \ + `extern` block") + .note("for more information, visit \ + https://doc.rust-lang.org/std/keyword.extern.html") + .emit(); + } + Err(mut err) => { + err.cancel(); + mem::replace(self, parser_snapshot); + self.expect(&token::Semi)?; + } + } + } else { + self.bump(); + } + Ok(()) + } + + /// Consume alternative await syntaxes like `await!(<expr>)`, `await <expr>`, + /// `await? <expr>`, `await(<expr>)`, and `await { <expr> }`. + crate fn parse_incorrect_await_syntax( + &mut self, + lo: Span, + await_sp: Span, + ) -> PResult<'a, (Span, ExprKind)> { + if self.token == token::Not { + // Handle `await!(<expr>)`. + self.expect(&token::Not)?; + self.expect(&token::OpenDelim(token::Paren))?; + let expr = self.parse_expr()?; + self.expect(&token::CloseDelim(token::Paren))?; + let sp = self.error_on_incorrect_await(lo, self.prev_span, &expr, false); + return Ok((sp, ExprKind::Await(expr))) + } + + let is_question = self.eat(&token::Question); // Handle `await? <expr>`. + let expr = if self.token == token::OpenDelim(token::Brace) { + // Handle `await { <expr> }`. + // This needs to be handled separatedly from the next arm to avoid + // interpreting `await { <expr> }?` as `<expr>?.await`. + self.parse_block_expr( + None, + self.token.span, + BlockCheckMode::Default, + ThinVec::new(), + ) + } else { + self.parse_expr() + }.map_err(|mut err| { + err.span_label(await_sp, "while parsing this incorrect await expression"); + err + })?; + let sp = self.error_on_incorrect_await(lo, expr.span, &expr, is_question); + Ok((sp, ExprKind::Await(expr))) + } + + fn error_on_incorrect_await(&self, lo: Span, hi: Span, expr: &Expr, is_question: bool) -> Span { + let expr_str = self.span_to_snippet(expr.span) + .unwrap_or_else(|_| pprust::expr_to_string(&expr)); + let suggestion = format!("{}.await{}", expr_str, if is_question { "?" } else { "" }); + let sp = lo.to(hi); + let app = match expr.node { + ExprKind::Try(_) => Applicability::MaybeIncorrect, // `await <expr>?` + _ => Applicability::MachineApplicable, + }; + self.struct_span_err(sp, "incorrect use of `await`") + .span_suggestion(sp, "`await` is a postfix operation", suggestion, app) + .emit(); + sp + } + + /// If encountering `future.await()`, consume and emit error. + crate fn recover_from_await_method_call(&mut self) { + if self.token == token::OpenDelim(token::Paren) && + self.look_ahead(1, |t| t == &token::CloseDelim(token::Paren)) + { + // future.await() + let lo = self.token.span; + self.bump(); // ( + let sp = lo.to(self.token.span); + self.bump(); // ) + self.struct_span_err(sp, "incorrect use of `await`") + .span_suggestion( + sp, + "`await` is not a method call, remove the parentheses", + String::new(), + Applicability::MachineApplicable, + ).emit() + } + } + + /// Recover a situation like `for ( $pat in $expr )` + /// and suggest writing `for $pat in $expr` instead. + /// + /// This should be called before parsing the `$block`. + crate fn recover_parens_around_for_head( + &mut self, + pat: P<Pat>, + expr: &Expr, + begin_paren: Option<Span>, + ) -> P<Pat> { + match (&self.token.kind, begin_paren) { + (token::CloseDelim(token::Paren), Some(begin_par_sp)) => { + self.bump(); + + let pat_str = self + // Remove the `(` from the span of the pattern: + .span_to_snippet(pat.span.trim_start(begin_par_sp).unwrap()) + .unwrap_or_else(|_| pprust::pat_to_string(&pat)); + + self.struct_span_err(self.prev_span, "unexpected closing `)`") + .span_label(begin_par_sp, "opening `(`") + .span_suggestion( + begin_par_sp.to(self.prev_span), + "remove parenthesis in `for` loop", + format!("{} in {}", pat_str, pprust::expr_to_string(&expr)), + // With e.g. `for (x) in y)` this would replace `(x) in y)` + // with `x) in y)` which is syntactically invalid. + // However, this is prevented before we get here. + Applicability::MachineApplicable, + ) + .emit(); + + // Unwrap `(pat)` into `pat` to avoid the `unused_parens` lint. + pat.and_then(|pat| match pat.node { + PatKind::Paren(pat) => pat, + _ => P(pat), + }) + } + _ => pat, + } + } + + crate fn could_ascription_be_path(&self, node: &ast::ExprKind) -> bool { + self.token.is_ident() && + if let ast::ExprKind::Path(..) = node { true } else { false } && + !self.token.is_reserved_ident() && // v `foo:bar(baz)` + self.look_ahead(1, |t| t == &token::OpenDelim(token::Paren)) || + self.look_ahead(1, |t| t == &token::Lt) && // `foo:bar<baz` + self.look_ahead(2, |t| t.is_ident()) || + self.look_ahead(1, |t| t == &token::Colon) && // `foo:bar:baz` + self.look_ahead(2, |t| t.is_ident()) || + self.look_ahead(1, |t| t == &token::ModSep) && + (self.look_ahead(2, |t| t.is_ident()) || // `foo:bar::baz` + self.look_ahead(2, |t| t == &token::Lt)) // `foo:bar::<baz>` + } + + crate fn recover_seq_parse_error( + &mut self, + delim: token::DelimToken, + lo: Span, + result: PResult<'a, P<Expr>>, + ) -> P<Expr> { + match result { + Ok(x) => x, + Err(mut err) => { + err.emit(); + // recover from parse error + self.consume_block(delim); + self.mk_expr(lo.to(self.prev_span), ExprKind::Err, ThinVec::new()) + } + } + } + + crate fn recover_closing_delimiter( + &mut self, + tokens: &[TokenKind], + mut err: DiagnosticBuilder<'a>, + ) -> PResult<'a, bool> { + let mut pos = None; + // we want to use the last closing delim that would apply + for (i, unmatched) in self.unclosed_delims.iter().enumerate().rev() { + if tokens.contains(&token::CloseDelim(unmatched.expected_delim)) + && Some(self.token.span) > unmatched.unclosed_span + { + pos = Some(i); + } + } + match pos { + Some(pos) => { + // Recover and assume that the detected unclosed delimiter was meant for + // this location. Emit the diagnostic and act as if the delimiter was + // present for the parser's sake. + + // Don't attempt to recover from this unclosed delimiter more than once. + let unmatched = self.unclosed_delims.remove(pos); + let delim = TokenType::Token(token::CloseDelim(unmatched.expected_delim)); + + // We want to suggest the inclusion of the closing delimiter where it makes + // the most sense, which is immediately after the last token: + // + // {foo(bar {}} + // - ^ + // | | + // | help: `)` may belong here + // | + // unclosed delimiter + if let Some(sp) = unmatched.unclosed_span { + err.span_label(sp, "unclosed delimiter"); + } + err.span_suggestion_short( + self.sess.source_map().next_point(self.prev_span), + &format!("{} may belong here", delim.to_string()), + delim.to_string(), + Applicability::MaybeIncorrect, + ); + err.emit(); + self.expected_tokens.clear(); // reduce errors + Ok(true) + } + _ => Err(err), + } + } + + /// Recover from `pub` keyword in places where it seems _reasonable_ but isn't valid. + crate fn eat_bad_pub(&mut self) { + if self.token.is_keyword(kw::Pub) { + match self.parse_visibility(false) { + Ok(vis) => { + self.diagnostic() + .struct_span_err(vis.span, "unnecessary visibility qualifier") + .span_label(vis.span, "`pub` not permitted here") + .emit(); + } + Err(mut err) => err.emit(), + } + } + } + + // Eat tokens until we can be relatively sure we reached the end of the + // statement. This is something of a best-effort heuristic. + // + // We terminate when we find an unmatched `}` (without consuming it). + crate fn recover_stmt(&mut self) { + self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore) + } + + // If `break_on_semi` is `Break`, then we will stop consuming tokens after + // finding (and consuming) a `;` outside of `{}` or `[]` (note that this is + // approximate - it can mean we break too early due to macros, but that + // should only lead to sub-optimal recovery, not inaccurate parsing). + // + // If `break_on_block` is `Break`, then we will stop consuming tokens + // after finding (and consuming) a brace-delimited block. + crate fn recover_stmt_(&mut self, break_on_semi: SemiColonMode, break_on_block: BlockMode) { + let mut brace_depth = 0; + let mut bracket_depth = 0; + let mut in_block = false; + debug!("recover_stmt_ enter loop (semi={:?}, block={:?})", + break_on_semi, break_on_block); + loop { + debug!("recover_stmt_ loop {:?}", self.token); + match self.token.kind { + token::OpenDelim(token::DelimToken::Brace) => { + brace_depth += 1; + self.bump(); + if break_on_block == BlockMode::Break && + brace_depth == 1 && + bracket_depth == 0 { + in_block = true; + } + } + token::OpenDelim(token::DelimToken::Bracket) => { + bracket_depth += 1; + self.bump(); + } + token::CloseDelim(token::DelimToken::Brace) => { + if brace_depth == 0 { + debug!("recover_stmt_ return - close delim {:?}", self.token); + break; + } + brace_depth -= 1; + self.bump(); + if in_block && bracket_depth == 0 && brace_depth == 0 { + debug!("recover_stmt_ return - block end {:?}", self.token); + break; + } + } + token::CloseDelim(token::DelimToken::Bracket) => { + bracket_depth -= 1; + if bracket_depth < 0 { + bracket_depth = 0; + } + self.bump(); + } + token::Eof => { + debug!("recover_stmt_ return - Eof"); + break; + } + token::Semi => { + self.bump(); + if break_on_semi == SemiColonMode::Break && + brace_depth == 0 && + bracket_depth == 0 { + debug!("recover_stmt_ return - Semi"); + break; + } + } + token::Comma if break_on_semi == SemiColonMode::Comma && + brace_depth == 0 && + bracket_depth == 0 => + { + debug!("recover_stmt_ return - Semi"); + break; + } + _ => { + self.bump() + } + } + } + } + + crate fn check_for_for_in_in_typo(&mut self, in_span: Span) { + if self.eat_keyword(kw::In) { + // a common typo: `for _ in in bar {}` + self.struct_span_err(self.prev_span, "expected iterable, found keyword `in`") + .span_suggestion_short( + in_span.until(self.prev_span), + "remove the duplicated `in`", + String::new(), + Applicability::MachineApplicable, + ) + .emit(); + } + } + + crate fn expected_semi_or_open_brace(&mut self) -> PResult<'a, ast::TraitItem> { + let token_str = self.this_token_descr(); + let mut err = self.fatal(&format!("expected `;` or `{{`, found {}", token_str)); + err.span_label(self.token.span, "expected `;` or `{`"); + Err(err) + } + + crate fn eat_incorrect_doc_comment_for_arg_type(&mut self) { + if let token::DocComment(_) = self.token.kind { + self.struct_span_err( + self.token.span, + "documentation comments cannot be applied to a function parameter's type", + ) + .span_label(self.token.span, "doc comments are not allowed here") + .emit(); + self.bump(); + } else if self.token == token::Pound && self.look_ahead(1, |t| { + *t == token::OpenDelim(token::Bracket) + }) { + let lo = self.token.span; + // Skip every token until next possible arg. + while self.token != token::CloseDelim(token::Bracket) { + self.bump(); + } + let sp = lo.to(self.token.span); + self.bump(); + self.struct_span_err( + sp, + "attributes cannot be applied to a function parameter's type", + ) + .span_label(sp, "attributes are not allowed here") + .emit(); + } + } + + crate fn argument_without_type( + &mut self, + err: &mut DiagnosticBuilder<'_>, + pat: P<ast::Pat>, + require_name: bool, + is_trait_item: bool, + ) -> Option<Ident> { + // If we find a pattern followed by an identifier, it could be an (incorrect) + // C-style parameter declaration. + if self.check_ident() && self.look_ahead(1, |t| { + *t == token::Comma || *t == token::CloseDelim(token::Paren) + }) { // `fn foo(String s) {}` + let ident = self.parse_ident().unwrap(); + let span = pat.span.with_hi(ident.span.hi()); + + err.span_suggestion( + span, + "declare the type after the parameter binding", + String::from("<identifier>: <type>"), + Applicability::HasPlaceholders, + ); + return Some(ident); + } else if let PatKind::Ident(_, ident, _) = pat.node { + if require_name && ( + is_trait_item || + self.token == token::Comma || + self.token == token::CloseDelim(token::Paren) + ) { // `fn foo(a, b) {}` or `fn foo(usize, usize) {}` + err.span_suggestion( + pat.span, + "if this was a parameter name, give it a type", + format!("{}: TypeName", ident), + Applicability::HasPlaceholders, + ); + err.span_suggestion( + pat.span, + "if this is a type, explicitly ignore the parameter name", + format!("_: {}", ident), + Applicability::MachineApplicable, + ); + err.note("anonymous parameters are removed in the 2018 edition (see RFC 1685)"); + return Some(ident); + } + } + None + } + + crate fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> { + let pat = self.parse_pat(Some("argument name"))?; + self.expect(&token::Colon)?; + let ty = self.parse_ty()?; + + self.diagnostic() + .struct_span_err_with_code( + pat.span, + "patterns aren't allowed in methods without bodies", + DiagnosticId::Error("E0642".into()), + ) + .span_suggestion_short( + pat.span, + "give this argument a name or use an underscore to ignore it", + "_".to_owned(), + Applicability::MachineApplicable, + ) + .emit(); + + // Pretend the pattern is `_`, to avoid duplicate errors from AST validation. + let pat = P(Pat { + node: PatKind::Wild, + span: pat.span, + id: ast::DUMMY_NODE_ID + }); + Ok((pat, ty)) + } + + crate fn recover_bad_self_arg( + &mut self, + mut arg: ast::Arg, + is_trait_item: bool, + ) -> PResult<'a, ast::Arg> { + let sp = arg.pat.span; + arg.ty.node = TyKind::Err; + let mut err = self.struct_span_err(sp, "unexpected `self` parameter in function"); + if is_trait_item { + err.span_label(sp, "must be the first associated function parameter"); + } else { + err.span_label(sp, "not valid as function parameter"); + err.note("`self` is only valid as the first parameter of an associated function"); + } + err.emit(); + Ok(arg) + } + + crate fn consume_block(&mut self, delim: token::DelimToken) { + let mut brace_depth = 0; + loop { + if self.eat(&token::OpenDelim(delim)) { + brace_depth += 1; + } else if self.eat(&token::CloseDelim(delim)) { + if brace_depth == 0 { + return; + } else { + brace_depth -= 1; + continue; + } + } else if self.token == token::Eof || self.eat(&token::CloseDelim(token::NoDelim)) { + return; + } else { + self.bump(); + } + } + } + + crate fn expected_expression_found(&self) -> DiagnosticBuilder<'a> { + let (span, msg) = match (&self.token.kind, self.subparser_name) { + (&token::Eof, Some(origin)) => { + let sp = self.sess.source_map().next_point(self.token.span); + (sp, format!("expected expression, found end of {}", origin)) + } + _ => (self.token.span, format!( + "expected expression, found {}", + self.this_token_descr(), + )), + }; + let mut err = self.struct_span_err(span, &msg); + let sp = self.sess.source_map().start_point(self.token.span); + if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow().get(&sp) { + self.sess.expr_parentheses_needed(&mut err, *sp, None); + } + err.span_label(span, "expected expression"); + err + } + + /// Replace duplicated recovered arguments with `_` pattern to avoid unecessary errors. + /// + /// This is necessary because at this point we don't know whether we parsed a function with + /// anonymous arguments or a function with names but no types. In order to minimize + /// unecessary errors, we assume the arguments are in the shape of `fn foo(a, b, c)` where + /// the arguments are *names* (so we don't emit errors about not being able to find `b` in + /// the local scope), but if we find the same name multiple times, like in `fn foo(i8, i8)`, + /// we deduplicate them to not complain about duplicated argument names. + crate fn deduplicate_recovered_arg_names(&self, fn_inputs: &mut Vec<Arg>) { + let mut seen_inputs = FxHashSet::default(); + for input in fn_inputs.iter_mut() { + let opt_ident = if let (PatKind::Ident(_, ident, _), TyKind::Err) = ( + &input.pat.node, &input.ty.node, + ) { + Some(*ident) + } else { + None + }; + if let Some(ident) = opt_ident { + if seen_inputs.contains(&ident) { + input.pat.node = PatKind::Wild; + } + seen_inputs.insert(ident); + } + } + } +} diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index 74fff3324ea..5121a9ef7b5 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -2,16 +2,16 @@ pub use CommentStyle::*; use crate::ast; use crate::source_map::SourceMap; -use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace}; -use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan}; -use crate::print::pprust; +use crate::parse::lexer::is_block_doc_comment; +use crate::parse::lexer::ParseSess; use syntax_pos::{BytePos, CharPos, Pos, FileName}; -use log::debug; -use std::io::Read; use std::usize; +#[cfg(test)] +mod tests; + #[derive(Clone, Copy, PartialEq, Debug)] pub enum CommentStyle { /// No code on either side of each line of the comment @@ -137,66 +137,6 @@ pub fn strip_doc_comment_decoration(comment: &str) -> String { panic!("not a doc-comment: {}", comment); } -fn push_blank_line_comment(rdr: &StringReader<'_>, comments: &mut Vec<Comment>) { - debug!(">>> blank-line comment"); - comments.push(Comment { - style: BlankLine, - lines: Vec::new(), - pos: rdr.pos, - }); -} - -fn consume_whitespace_counting_blank_lines( - rdr: &mut StringReader<'_>, - comments: &mut Vec<Comment> -) { - while is_pattern_whitespace(rdr.ch) && !rdr.is_eof() { - if rdr.ch_is('\n') { - push_blank_line_comment(rdr, &mut *comments); - } - rdr.bump(); - } -} - -fn read_shebang_comment(rdr: &mut StringReader<'_>, - code_to_the_left: bool, - comments: &mut Vec<Comment>) { - debug!(">>> shebang comment"); - let p = rdr.pos; - debug!("<<< shebang comment"); - comments.push(Comment { - style: if code_to_the_left { Trailing } else { Isolated }, - lines: vec![rdr.read_one_line_comment()], - pos: p, - }); -} - -fn read_line_comments(rdr: &mut StringReader<'_>, - code_to_the_left: bool, - comments: &mut Vec<Comment>) { - debug!(">>> line comments"); - let p = rdr.pos; - let mut lines: Vec<String> = Vec::new(); - while rdr.ch_is('/') && rdr.nextch_is('/') { - let line = rdr.read_one_line_comment(); - debug!("{}", line); - // Doc comments are not put in comments. - if is_doc_comment(&line[..]) { - break; - } - lines.push(line); - rdr.consume_non_eol_whitespace(); - } - debug!("<<< line comments"); - if !lines.is_empty() { - comments.push(Comment { - style: if code_to_the_left { Trailing } else { Isolated }, - lines, - pos: p, - }); - } -} - /// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. /// Otherwise returns `Some(k)` where `k` is first char offset after that leading /// whitespace. Note that `k` may be outside bounds of `s`. @@ -211,246 +151,104 @@ fn all_whitespace(s: &str, col: CharPos) -> Option<usize> { Some(idx) } -fn trim_whitespace_prefix_and_push_line(lines: &mut Vec<String>, s: String, col: CharPos) { +fn trim_whitespace_prefix(s: &str, col: CharPos) -> &str { let len = s.len(); - let s1 = match all_whitespace(&s[..], col) { - Some(col) => { - if col < len { - s[col..len].to_string() - } else { - String::new() - } - } + match all_whitespace(&s, col) { + Some(col) => if col < len { &s[col..] } else { "" }, None => s, - }; - debug!("pushing line: {}", s1); - lines.push(s1); -} - -fn read_block_comment(rdr: &mut StringReader<'_>, - code_to_the_left: bool, - comments: &mut Vec<Comment>) { - debug!(">>> block comment"); - let p = rdr.pos; - let mut lines: Vec<String> = Vec::new(); - - // Count the number of chars since the start of the line by rescanning. - let src_index = rdr.src_index(rdr.source_file.line_begin_pos(rdr.pos)); - let end_src_index = rdr.src_index(rdr.pos); - assert!(src_index <= end_src_index, - "src_index={}, end_src_index={}, line_begin_pos={}", - src_index, end_src_index, rdr.source_file.line_begin_pos(rdr.pos).to_u32()); - - let col = CharPos(rdr.src[src_index..end_src_index].chars().count()); - - rdr.bump(); - rdr.bump(); - - let mut curr_line = String::from("/*"); - - // doc-comments are not really comments, they are attributes - if (rdr.ch_is('*') && !rdr.nextch_is('*')) || rdr.ch_is('!') { - while !(rdr.ch_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() { - curr_line.push(rdr.ch.unwrap()); - rdr.bump(); - } - if !rdr.is_eof() { - curr_line.push_str("*/"); - rdr.bump(); - rdr.bump(); - } - if is_block_doc_comment(&curr_line[..]) { - return; - } - assert!(!curr_line.contains('\n')); - lines.push(curr_line); - } else { - let mut level: isize = 1; - while level > 0 { - debug!("=== block comment level {}", level); - if rdr.is_eof() { - rdr.fatal("unterminated block comment").raise(); - } - if rdr.ch_is('\n') { - trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col); - curr_line = String::new(); - rdr.bump(); - } else { - curr_line.push(rdr.ch.unwrap()); - if rdr.ch_is('/') && rdr.nextch_is('*') { - rdr.bump(); - rdr.bump(); - curr_line.push('*'); - level += 1; - } else { - if rdr.ch_is('*') && rdr.nextch_is('/') { - rdr.bump(); - rdr.bump(); - curr_line.push('/'); - level -= 1; - } else { - rdr.bump(); - } - } - } - } - if !curr_line.is_empty() { - trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col); - } - } - - let mut style = if code_to_the_left { - Trailing - } else { - Isolated - }; - rdr.consume_non_eol_whitespace(); - if !rdr.is_eof() && !rdr.ch_is('\n') && lines.len() == 1 { - style = Mixed; } - debug!("<<< block comment"); - comments.push(Comment { - style, - lines, - pos: p, - }); } - -fn consume_comment(rdr: &mut StringReader<'_>, - comments: &mut Vec<Comment>, - code_to_the_left: &mut bool, - anything_to_the_left: &mut bool) { - debug!(">>> consume comment"); - if rdr.ch_is('/') && rdr.nextch_is('/') { - read_line_comments(rdr, *code_to_the_left, comments); - *code_to_the_left = false; - *anything_to_the_left = false; - } else if rdr.ch_is('/') && rdr.nextch_is('*') { - read_block_comment(rdr, *code_to_the_left, comments); - *anything_to_the_left = true; - } else if rdr.ch_is('#') && rdr.nextch_is('!') { - read_shebang_comment(rdr, *code_to_the_left, comments); - *code_to_the_left = false; - *anything_to_the_left = false; - } else { - panic!(); +fn split_block_comment_into_lines( + text: &str, + col: CharPos, +) -> Vec<String> { + let mut res: Vec<String> = vec![]; + let mut lines = text.lines(); + // just push the first line + res.extend(lines.next().map(|it| it.to_string())); + // for other lines, strip common whitespace prefix + for line in lines { + res.push(trim_whitespace_prefix(line, col).to_string()) } - debug!("<<< consume comment"); -} - -#[derive(Clone)] -pub struct Literal { - pub lit: String, - pub pos: BytePos, + res } // it appears this function is called only from pprust... that's // probably not a good thing. -pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) - -> (Vec<Comment>, Vec<Literal>) -{ - let mut src = String::new(); - srdr.read_to_string(&mut src).unwrap(); +pub fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec<Comment> { let cm = SourceMap::new(sess.source_map().path_mapping().clone()); let source_file = cm.new_source_file(path, src); - let mut rdr = lexer::StringReader::new_raw(sess, source_file, None); + let text = (*source_file.src.as_ref().unwrap()).clone(); + let text: &str = text.as_str(); + let start_bpos = source_file.start_pos; + let mut pos = 0; let mut comments: Vec<Comment> = Vec::new(); - let mut literals: Vec<Literal> = Vec::new(); - let mut code_to_the_left = false; // Only code - let mut anything_to_the_left = false; // Code or comments + let mut code_to_the_left = false; - while !rdr.is_eof() { - loop { - // Eat all the whitespace and count blank lines. - rdr.consume_non_eol_whitespace(); - if rdr.ch_is('\n') { - if anything_to_the_left { - rdr.bump(); // The line is not blank, do not count. + if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { + comments.push(Comment { + style: Isolated, + lines: vec![text[..shebang_len].to_string()], + pos: start_bpos, + }); + pos += shebang_len; + } + + for token in rustc_lexer::tokenize(&text[pos..]) { + let token_text = &text[pos..pos + token.len]; + match token.kind { + rustc_lexer::TokenKind::Whitespace => { + if let Some(mut idx) = token_text.find('\n') { + code_to_the_left = false; + while let Some(next_newline) = &token_text[idx + 1..].find('\n') { + idx = idx + 1 + next_newline; + comments.push(Comment { + style: BlankLine, + lines: vec![], + pos: start_bpos + BytePos((pos + idx) as u32), + }); + } } - consume_whitespace_counting_blank_lines(&mut rdr, &mut comments); - code_to_the_left = false; - anything_to_the_left = false; } - // Eat one comment group - if rdr.peeking_at_comment() { - consume_comment(&mut rdr, &mut comments, - &mut code_to_the_left, &mut anything_to_the_left); - } else { - break + rustc_lexer::TokenKind::BlockComment { terminated: _ } => { + if !is_block_doc_comment(token_text) { + let code_to_the_right = match text[pos + token.len..].chars().next() { + Some('\r') | Some('\n') => false, + _ => true, + }; + let style = match (code_to_the_left, code_to_the_right) { + (true, true) | (false, true) => Mixed, + (false, false) => Isolated, + (true, false) => Trailing, + }; + + // Count the number of chars since the start of the line by rescanning. + let pos_in_file = start_bpos + BytePos(pos as u32); + let line_begin_in_file = source_file.line_begin_pos(pos_in_file); + let line_begin_pos = (line_begin_in_file - start_bpos).to_usize(); + let col = CharPos(text[line_begin_pos..pos].chars().count()); + + let lines = split_block_comment_into_lines(token_text, col); + comments.push(Comment { style, lines, pos: pos_in_file }) + } + } + rustc_lexer::TokenKind::LineComment => { + if !is_doc_comment(token_text) { + comments.push(Comment { + style: if code_to_the_left { Trailing } else { Isolated }, + lines: vec![token_text.to_string()], + pos: start_bpos + BytePos(pos as u32), + }) + } + } + _ => { + code_to_the_left = true; } } - - let bstart = rdr.pos; - rdr.next_token(); - // discard, and look ahead; we're working with internal state - let TokenAndSpan { tok, sp } = rdr.peek(); - if tok.is_lit() { - rdr.with_str_from(bstart, |s| { - debug!("tok lit: {}", s); - literals.push(Literal { - lit: s.to_string(), - pos: sp.lo(), - }); - }) - } else { - debug!("tok: {}", pprust::token_to_string(&tok)); - } - code_to_the_left = true; - anything_to_the_left = true; + pos += token.len; } - (comments, literals) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_block_doc_comment_1() { - let comment = "/**\n * Test \n ** Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test \n* Test\n Test"); - } - - #[test] - fn test_block_doc_comment_2() { - let comment = "/**\n * Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test\n Test"); - } - - #[test] - fn test_block_doc_comment_3() { - let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " let a: *i32;\n *a = 5;"); - } - - #[test] - fn test_block_doc_comment_4() { - let comment = "/*******************\n test\n *********************/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " test"); - } - - #[test] - fn test_line_doc_comment() { - let stripped = strip_doc_comment_decoration("/// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///! test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("///!test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("//test"); - assert_eq!(stripped, "test"); - } + comments } diff --git a/src/libsyntax/parse/lexer/comments/tests.rs b/src/libsyntax/parse/lexer/comments/tests.rs new file mode 100644 index 00000000000..f9cd69fb50d --- /dev/null +++ b/src/libsyntax/parse/lexer/comments/tests.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn test_block_doc_comment_1() { + let comment = "/**\n * Test \n ** Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test \n* Test\n Test"); +} + +#[test] +fn test_block_doc_comment_2() { + let comment = "/**\n * Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test\n Test"); +} + +#[test] +fn test_block_doc_comment_3() { + let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " let a: *i32;\n *a = 5;"); +} + +#[test] +fn test_block_doc_comment_4() { + let comment = "/*******************\n test\n *********************/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " test"); +} + +#[test] +fn test_line_doc_comment() { + let stripped = strip_doc_comment_decoration("/// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///! test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("///!test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("//test"); + assert_eq!(stripped, "test"); +} diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 01e3b292903..e86d4c7fde6 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,39 +1,28 @@ -use crate::ast::{self, Ident}; -use crate::source_map::{SourceMap, FilePathMapping}; -use crate::parse::{token, ParseSess}; -use crate::symbol::{Symbol, keywords}; +use crate::parse::ParseSess; +use crate::parse::token::{self, Token, TokenKind}; +use crate::symbol::{sym, Symbol}; +use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; -use errors::{Applicability, FatalError, Diagnostic, DiagnosticBuilder}; -use syntax_pos::{BytePos, CharPos, Pos, Span, NO_EXPANSION}; -use core::unicode::property::Pattern_White_Space; +use errors::{FatalError, DiagnosticBuilder}; +use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION}; +use rustc_lexer::Base; +use rustc_lexer::unescape; use std::borrow::Cow; use std::char; use std::iter; -use std::mem::replace; +use std::convert::TryInto; use rustc_data_structures::sync::Lrc; use log::debug; +#[cfg(test)] +mod tests; + pub mod comments; mod tokentrees; mod unicode_chars; #[derive(Clone, Debug)] -pub struct TokenAndSpan { - pub tok: token::Token, - pub sp: Span, -} - -impl Default for TokenAndSpan { - fn default() -> Self { - TokenAndSpan { - tok: token::Whitespace, - sp: syntax_pos::DUMMY_SP, - } - } -} - -#[derive(Clone, Debug)] pub struct UnmatchedBrace { pub expected_delim: token::DelimToken, pub found_delim: token::DelimToken, @@ -43,193 +32,22 @@ pub struct UnmatchedBrace { } pub struct StringReader<'a> { - pub sess: &'a ParseSess, - /// The absolute offset within the source_map of the next character to read - pub next_pos: BytePos, - /// The absolute offset within the source_map of the current character - pub pos: BytePos, - /// The current character (which has been read from self.pos) - pub ch: Option<char>, - pub source_file: Lrc<syntax_pos::SourceFile>, + sess: &'a ParseSess, + /// Initial position, read-only. + start_pos: BytePos, + /// The absolute offset within the source_map of the current character. + pos: BytePos, /// Stop reading src at this index. - pub end_src_index: usize, - // cached: - peek_tok: token::Token, - peek_span: Span, - peek_span_src_raw: Span, - fatal_errs: Vec<DiagnosticBuilder<'a>>, - // cache a direct reference to the source text, so that we don't have to - // retrieve it via `self.source_file.src.as_ref().unwrap()` all the time. + end_src_index: usize, + /// Source text to tokenize. src: Lrc<String>, - token: token::Token, - span: Span, - /// The raw source span which *does not* take `override_span` into account - span_src_raw: Span, - /// Stack of open delimiters and their spans. Used for error message. - open_braces: Vec<(token::DelimToken, Span)>, - crate unmatched_braces: Vec<UnmatchedBrace>, - /// The type and spans for all braces - /// - /// Used only for error recovery when arriving to EOF with mismatched braces. - matching_delim_spans: Vec<(token::DelimToken, Span, Span)>, - crate override_span: Option<Span>, - last_unclosed_found_span: Option<Span>, + override_span: Option<Span>, } impl<'a> StringReader<'a> { - fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span { - self.mk_sp_and_raw(lo, hi).0 - } - - fn mk_sp_and_raw(&self, lo: BytePos, hi: BytePos) -> (Span, Span) { - let raw = Span::new(lo, hi, NO_EXPANSION); - let real = self.override_span.unwrap_or(raw); - - (real, raw) - } - - fn mk_ident(&self, string: &str) -> Ident { - let mut ident = Ident::from_str(string); - if let Some(span) = self.override_span { - ident.span = span; - } - - ident - } - - fn unwrap_or_abort(&mut self, res: Result<TokenAndSpan, ()>) -> TokenAndSpan { - match res { - Ok(tok) => tok, - Err(_) => { - self.emit_fatal_errors(); - FatalError.raise(); - } - } - } - - fn next_token(&mut self) -> TokenAndSpan where Self: Sized { - let res = self.try_next_token(); - self.unwrap_or_abort(res) - } - - /// Returns the next token. EFFECT: advances the string_reader. - pub fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> { - assert!(self.fatal_errs.is_empty()); - let ret_val = TokenAndSpan { - tok: replace(&mut self.peek_tok, token::Whitespace), - sp: self.peek_span, - }; - self.advance_token()?; - self.span_src_raw = self.peek_span_src_raw; - - Ok(ret_val) - } - - /// Immutably extract string if found at current position with given delimiters - pub fn peek_delimited(&self, from_ch: char, to_ch: char) -> Option<String> { - let mut pos = self.pos; - let mut idx = self.src_index(pos); - let mut ch = char_at(&self.src, idx); - if ch != from_ch { - return None; - } - pos = pos + Pos::from_usize(ch.len_utf8()); - let start_pos = pos; - idx = self.src_index(pos); - while idx < self.end_src_index { - ch = char_at(&self.src, idx); - if ch == to_ch { - return Some(self.src[self.src_index(start_pos)..self.src_index(pos)].to_string()); - } - pos = pos + Pos::from_usize(ch.len_utf8()); - idx = self.src_index(pos); - } - return None; - } - - fn try_real_token(&mut self) -> Result<TokenAndSpan, ()> { - let mut t = self.try_next_token()?; - loop { - match t.tok { - token::Whitespace | token::Comment | token::Shebang(_) => { - t = self.try_next_token()?; - } - _ => break, - } - } - - self.token = t.tok.clone(); - self.span = t.sp; - - Ok(t) - } - - pub fn real_token(&mut self) -> TokenAndSpan { - let res = self.try_real_token(); - self.unwrap_or_abort(res) - } - - #[inline] - fn is_eof(&self) -> bool { - self.ch.is_none() - } - - fn fail_unterminated_raw_string(&self, pos: BytePos, hash_count: u16) { - let mut err = self.struct_span_fatal(pos, pos, "unterminated raw string"); - err.span_label(self.mk_sp(pos, pos), "unterminated raw string"); - - if hash_count > 0 { - err.note(&format!("this raw string should be terminated with `\"{}`", - "#".repeat(hash_count as usize))); - } - - err.emit(); - FatalError.raise(); - } - - fn fatal(&self, m: &str) -> FatalError { - self.fatal_span(self.peek_span, m) - } - - pub fn emit_fatal_errors(&mut self) { - for err in &mut self.fatal_errs { - err.emit(); - } - - self.fatal_errs.clear(); - } - - pub fn buffer_fatal_errors(&mut self) -> Vec<Diagnostic> { - let mut buffer = Vec::new(); - - for err in self.fatal_errs.drain(..) { - err.buffer(&mut buffer); - } - - buffer - } - - pub fn peek(&self) -> TokenAndSpan { - // FIXME(pcwalton): Bad copy! - TokenAndSpan { - tok: self.peek_tok.clone(), - sp: self.peek_span, - } - } - - /// For comments.rs, which hackily pokes into next_pos and ch - fn new_raw(sess: &'a ParseSess, + pub fn new(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>, override_span: Option<Span>) -> Self { - let mut sr = StringReader::new_raw_internal(sess, source_file, override_span); - sr.bump(); - - sr - } - - fn new_raw_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile>, - override_span: Option<Span>) -> Self - { if source_file.src.is_none() { sess.span_diagnostic.bug(&format!("Cannot lex source_file without source: {}", source_file.name)); @@ -239,48 +57,11 @@ impl<'a> StringReader<'a> { StringReader { sess, - next_pos: source_file.start_pos, + start_pos: source_file.start_pos, pos: source_file.start_pos, - ch: Some('\n'), - source_file, end_src_index: src.len(), - // dummy values; not read - peek_tok: token::Eof, - peek_span: syntax_pos::DUMMY_SP, - peek_span_src_raw: syntax_pos::DUMMY_SP, src, - fatal_errs: Vec::new(), - token: token::Eof, - span: syntax_pos::DUMMY_SP, - span_src_raw: syntax_pos::DUMMY_SP, - open_braces: Vec::new(), - unmatched_braces: Vec::new(), - matching_delim_spans: Vec::new(), override_span, - last_unclosed_found_span: None, - } - } - - pub fn new(sess: &'a ParseSess, - source_file: Lrc<syntax_pos::SourceFile>, - override_span: Option<Span>) -> Self { - let mut sr = StringReader::new_raw(sess, source_file, override_span); - if sr.advance_token().is_err() { - sr.emit_fatal_errors(); - FatalError.raise(); - } - - sr - } - - pub fn new_or_buffered_errs(sess: &'a ParseSess, - source_file: Lrc<syntax_pos::SourceFile>, - override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> { - let mut sr = StringReader::new_raw(sess, source_file, override_span); - if sr.advance_token().is_err() { - Err(sr.buffer_fatal_errors()) - } else { - Ok(sr) } } @@ -293,25 +74,61 @@ impl<'a> StringReader<'a> { span = span.shrink_to_lo(); } - let mut sr = StringReader::new_raw_internal(sess, begin.sf, None); + let mut sr = StringReader::new(sess, begin.sf, None); // Seek the lexer to the right byte range. - sr.next_pos = span.lo(); sr.end_src_index = sr.src_index(span.hi()); - sr.bump(); + sr + } - if sr.advance_token().is_err() { - sr.emit_fatal_errors(); - FatalError.raise(); - } - sr + fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span { + self.override_span.unwrap_or_else(|| Span::new(lo, hi, NO_EXPANSION)) } - #[inline] - fn ch_is(&self, c: char) -> bool { - self.ch == Some(c) + /// Returns the next token, including trivia like whitespace or comments. + /// + /// `Err(())` means that some errors were encountered, which can be + /// retrieved using `buffer_fatal_errors`. + pub fn next_token(&mut self) -> Token { + let start_src_index = self.src_index(self.pos); + let text: &str = &self.src[start_src_index..self.end_src_index]; + + if text.is_empty() { + let span = self.mk_sp(self.pos, self.pos); + return Token::new(token::Eof, span); + } + + { + let is_beginning_of_file = self.pos == self.start_pos; + if is_beginning_of_file { + if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { + let start = self.pos; + self.pos = self.pos + BytePos::from_usize(shebang_len); + + let sym = self.symbol_from(start + BytePos::from_usize("#!".len())); + let kind = token::Shebang(sym); + + let span = self.mk_sp(start, self.pos); + return Token::new(kind, span); + } + } + } + + let token = rustc_lexer::first_token(text); + + let start = self.pos; + self.pos = self.pos + BytePos::from_usize(token.len); + + debug!("try_next_token: {:?}({:?})", token.kind, self.str_from(start)); + + // This could use `?`, but that makes code significantly (10-20%) slower. + // https://github.com/rust-lang/rust/issues/37939 + let kind = self.cook_lexer_token(token.kind, start); + + let span = self.mk_sp(start, self.pos); + Token::new(kind, span) } /// Report a fatal lexical error with a given span. @@ -335,29 +152,6 @@ impl<'a> StringReader<'a> { self.err_span(self.mk_sp(from_pos, to_pos), m) } - /// Pushes a character to a message string for error reporting - fn push_escaped_char_for_msg(m: &mut String, c: char) { - match c { - '\u{20}'..='\u{7e}' => { - // Don't escape \, ' or " for user-facing messages - m.push(c); - } - _ => { - m.extend(c.escape_default()); - } - } - } - - /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an - /// escaped character to the error message - fn fatal_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) -> FatalError { - let mut m = m.to_string(); - m.push_str(": "); - Self::push_escaped_char_for_msg(&mut m, c); - - self.fatal_span_(from_pos, to_pos, &m[..]) - } - fn struct_span_fatal(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> DiagnosticBuilder<'a> { @@ -369,104 +163,357 @@ impl<'a> StringReader<'a> { { let mut m = m.to_string(); m.push_str(": "); - Self::push_escaped_char_for_msg(&mut m, c); + push_escaped_char(&mut m, c); self.sess.span_diagnostic.struct_span_fatal(self.mk_sp(from_pos, to_pos), &m[..]) } - /// Report a lexical error spanning [`from_pos`, `to_pos`), appending an - /// escaped character to the error message - fn err_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) { - let mut m = m.to_string(); - m.push_str(": "); - Self::push_escaped_char_for_msg(&mut m, c); - self.err_span_(from_pos, to_pos, &m[..]); - } - - fn struct_err_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) - -> DiagnosticBuilder<'a> - { - let mut m = m.to_string(); - m.push_str(": "); - Self::push_escaped_char_for_msg(&mut m, c); + /// Turns simple `rustc_lexer::TokenKind` enum into a rich + /// `libsyntax::TokenKind`. This turns strings into interned + /// symbols and runs additional validation. + fn cook_lexer_token( + &self, + token: rustc_lexer::TokenKind, + start: BytePos, + ) -> TokenKind { + match token { + rustc_lexer::TokenKind::LineComment => { + let string = self.str_from(start); + // comments with only more "/"s are not doc comments + let tok = if is_doc_comment(string) { + let mut idx = 0; + loop { + idx = match string[idx..].find('\r') { + None => break, + Some(it) => idx + it + 1 + }; + if string[idx..].chars().next() != Some('\n') { + self.err_span_(start + BytePos(idx as u32 - 1), + start + BytePos(idx as u32), + "bare CR not allowed in doc-comment"); + } + } + token::DocComment(Symbol::intern(string)) + } else { + token::Comment + }; - self.sess.span_diagnostic.struct_span_err(self.mk_sp(from_pos, to_pos), &m[..]) - } + tok + } + rustc_lexer::TokenKind::BlockComment { terminated } => { + let string = self.str_from(start); + // block comments starting with "/**" or "/*!" are doc-comments + // but comments with only "*"s between two "/"s are not + let is_doc_comment = is_block_doc_comment(string); - /// Report a lexical error spanning [`from_pos`, `to_pos`), appending the - /// offending string to the error message - fn fatal_span_verbose(&self, from_pos: BytePos, to_pos: BytePos, mut m: String) -> FatalError { - m.push_str(": "); - m.push_str(&self.src[self.src_index(from_pos)..self.src_index(to_pos)]); + if !terminated { + let msg = if is_doc_comment { + "unterminated block doc-comment" + } else { + "unterminated block comment" + }; + let last_bpos = self.pos; + self.fatal_span_(start, last_bpos, msg).raise(); + } - self.fatal_span_(from_pos, to_pos, &m[..]) - } + let tok = if is_doc_comment { + let has_cr = string.contains('\r'); + let string = if has_cr { + self.translate_crlf(start, + string, + "bare CR not allowed in block doc-comment") + } else { + string.into() + }; + token::DocComment(Symbol::intern(&string[..])) + } else { + token::Comment + }; - /// Advance peek_tok and peek_span to refer to the next token, and - /// possibly update the interner. - fn advance_token(&mut self) -> Result<(), ()> { - match self.scan_whitespace_or_comment() { - Some(comment) => { - self.peek_span_src_raw = comment.sp; - self.peek_span = comment.sp; - self.peek_tok = comment.tok; + tok } - None => { - if self.is_eof() { - self.peek_tok = token::Eof; - let (real, raw) = self.mk_sp_and_raw( - self.source_file.end_pos, - self.source_file.end_pos, - ); - self.peek_span = real; - self.peek_span_src_raw = raw; + rustc_lexer::TokenKind::Whitespace => token::Whitespace, + rustc_lexer::TokenKind::Ident | rustc_lexer::TokenKind::RawIdent => { + let is_raw_ident = token == rustc_lexer::TokenKind::RawIdent; + let mut ident_start = start; + if is_raw_ident { + ident_start = ident_start + BytePos(2); + } + // FIXME: perform NFKC normalization here. (Issue #2253) + let sym = self.symbol_from(ident_start); + if is_raw_ident { + let span = self.mk_sp(start, self.pos); + if !sym.can_be_raw() { + self.err_span(span, &format!("`{}` cannot be a raw identifier", sym)); + } + self.sess.raw_identifier_spans.borrow_mut().push(span); + } + token::Ident(sym, is_raw_ident) + } + rustc_lexer::TokenKind::Literal { kind, suffix_start } => { + let suffix_start = start + BytePos(suffix_start as u32); + let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind); + let suffix = if suffix_start < self.pos { + let string = self.str_from(suffix_start); + if string == "_" { + self.sess.span_diagnostic + .struct_span_warn(self.mk_sp(suffix_start, self.pos), + "underscore literal suffix is not allowed") + .warn("this was previously accepted by the compiler but is \ + being phased out; it will become a hard error in \ + a future release!") + .note("for more information, see issue #42326 \ + <https://github.com/rust-lang/rust/issues/42326>") + .emit(); + None + } else { + Some(Symbol::intern(string)) + } } else { - let start_bytepos = self.pos; - self.peek_tok = self.next_token_inner()?; - let (real, raw) = self.mk_sp_and_raw(start_bytepos, self.pos); - self.peek_span = real; - self.peek_span_src_raw = raw; + None }; + token::Literal(token::Lit { kind, symbol, suffix }) + } + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + // Include the leading `'` in the real identifier, for macro + // expansion purposes. See #12512 for the gory details of why + // this is necessary. + let lifetime_name = self.str_from(start); + if starts_with_number { + self.err_span_( + start, + self.pos, + "lifetimes cannot start with a number", + ); + } + let ident = Symbol::intern(lifetime_name); + token::Lifetime(ident) + } + rustc_lexer::TokenKind::Semi => token::Semi, + rustc_lexer::TokenKind::Comma => token::Comma, + rustc_lexer::TokenKind::DotDotDot => token::DotDotDot, + rustc_lexer::TokenKind::DotDotEq => token::DotDotEq, + rustc_lexer::TokenKind::DotDot => token::DotDot, + rustc_lexer::TokenKind::Dot => token::Dot, + rustc_lexer::TokenKind::OpenParen => token::OpenDelim(token::Paren), + rustc_lexer::TokenKind::CloseParen => token::CloseDelim(token::Paren), + rustc_lexer::TokenKind::OpenBrace => token::OpenDelim(token::Brace), + rustc_lexer::TokenKind::CloseBrace => token::CloseDelim(token::Brace), + rustc_lexer::TokenKind::OpenBracket => token::OpenDelim(token::Bracket), + rustc_lexer::TokenKind::CloseBracket => token::CloseDelim(token::Bracket), + rustc_lexer::TokenKind::At => token::At, + rustc_lexer::TokenKind::Pound => token::Pound, + rustc_lexer::TokenKind::Tilde => token::Tilde, + rustc_lexer::TokenKind::Question => token::Question, + rustc_lexer::TokenKind::ColonColon => token::ModSep, + rustc_lexer::TokenKind::Colon => token::Colon, + rustc_lexer::TokenKind::Dollar => token::Dollar, + rustc_lexer::TokenKind::EqEq => token::EqEq, + rustc_lexer::TokenKind::Eq => token::Eq, + rustc_lexer::TokenKind::FatArrow => token::FatArrow, + rustc_lexer::TokenKind::Ne => token::Ne, + rustc_lexer::TokenKind::Not => token::Not, + rustc_lexer::TokenKind::Le => token::Le, + rustc_lexer::TokenKind::LArrow => token::LArrow, + rustc_lexer::TokenKind::Lt => token::Lt, + rustc_lexer::TokenKind::ShlEq => token::BinOpEq(token::Shl), + rustc_lexer::TokenKind::Shl => token::BinOp(token::Shl), + rustc_lexer::TokenKind::Ge => token::Ge, + rustc_lexer::TokenKind::Gt => token::Gt, + rustc_lexer::TokenKind::ShrEq => token::BinOpEq(token::Shr), + rustc_lexer::TokenKind::Shr => token::BinOp(token::Shr), + rustc_lexer::TokenKind::RArrow => token::RArrow, + rustc_lexer::TokenKind::Minus => token::BinOp(token::Minus), + rustc_lexer::TokenKind::MinusEq => token::BinOpEq(token::Minus), + rustc_lexer::TokenKind::And => token::BinOp(token::And), + rustc_lexer::TokenKind::AndEq => token::BinOpEq(token::And), + rustc_lexer::TokenKind::AndAnd => token::AndAnd, + rustc_lexer::TokenKind::Or => token::BinOp(token::Or), + rustc_lexer::TokenKind::OrEq => token::BinOpEq(token::Or), + rustc_lexer::TokenKind::OrOr => token::OrOr, + rustc_lexer::TokenKind::Plus => token::BinOp(token::Plus), + rustc_lexer::TokenKind::PlusEq => token::BinOpEq(token::Plus), + rustc_lexer::TokenKind::Star => token::BinOp(token::Star), + rustc_lexer::TokenKind::StarEq => token::BinOpEq(token::Star), + rustc_lexer::TokenKind::Slash => token::BinOp(token::Slash), + rustc_lexer::TokenKind::SlashEq => token::BinOpEq(token::Slash), + rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret), + rustc_lexer::TokenKind::CaretEq => token::BinOpEq(token::Caret), + rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent), + rustc_lexer::TokenKind::PercentEq => token::BinOpEq(token::Percent), + + rustc_lexer::TokenKind::Unknown => { + let c = self.str_from(start).chars().next().unwrap(); + let mut err = self.struct_fatal_span_char(start, + self.pos, + "unknown start of token", + c); + // FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, + // instead of keeping a table in `check_for_substitution`into the token. Ideally, + // this should be inside `rustc_lexer`. However, we should first remove compound + // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, + // as there will be less overall work to do this way. + let token = unicode_chars::check_for_substitution(self, start, c, &mut err) + .unwrap_or_else(|| token::Unknown(self.symbol_from(start))); + err.emit(); + token } } + } - Ok(()) + fn cook_lexer_literal( + &self, + start: BytePos, + suffix_start: BytePos, + kind: rustc_lexer::LiteralKind + ) -> (token::LitKind, Symbol) { + match kind { + rustc_lexer::LiteralKind::Char { terminated } => { + if !terminated { + self.fatal_span_(start, suffix_start, + "unterminated character literal".into()) + .raise() + } + let content_start = start + BytePos(1); + let content_end = suffix_start - BytePos(1); + self.validate_char_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::Char, id) + }, + rustc_lexer::LiteralKind::Byte { terminated } => { + if !terminated { + self.fatal_span_(start + BytePos(1), suffix_start, + "unterminated byte constant".into()) + .raise() + } + let content_start = start + BytePos(2); + let content_end = suffix_start - BytePos(1); + self.validate_byte_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::Byte, id) + }, + rustc_lexer::LiteralKind::Str { terminated } => { + if !terminated { + self.fatal_span_(start, suffix_start, + "unterminated double quote string".into()) + .raise() + } + let content_start = start + BytePos(1); + let content_end = suffix_start - BytePos(1); + self.validate_str_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::Str, id) + } + rustc_lexer::LiteralKind::ByteStr { terminated } => { + if !terminated { + self.fatal_span_(start + BytePos(1), suffix_start, + "unterminated double quote byte string".into()) + .raise() + } + let content_start = start + BytePos(2); + let content_end = suffix_start - BytePos(1); + self.validate_byte_str_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::ByteStr, id) + } + rustc_lexer::LiteralKind::RawStr { n_hashes, started, terminated } => { + if !started { + self.report_non_started_raw_string(start); + } + if !terminated { + self.report_unterminated_raw_string(start, n_hashes) + } + let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes); + let n = u32::from(n_hashes); + let content_start = start + BytePos(2 + n); + let content_end = suffix_start - BytePos(1 + n); + self.validate_raw_str_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::StrRaw(n_hashes), id) + } + rustc_lexer::LiteralKind::RawByteStr { n_hashes, started, terminated } => { + if !started { + self.report_non_started_raw_string(start); + } + if !terminated { + self.report_unterminated_raw_string(start, n_hashes) + } + let n_hashes: u16 = self.restrict_n_hashes(start, n_hashes); + let n = u32::from(n_hashes); + let content_start = start + BytePos(3 + n); + let content_end = suffix_start - BytePos(1 + n); + self.validate_raw_byte_str_escape(content_start, content_end); + let id = self.symbol_from_to(content_start, content_end); + (token::ByteStrRaw(n_hashes), id) + } + rustc_lexer::LiteralKind::Int { base, empty_int } => { + if empty_int { + self.err_span_(start, suffix_start, "no valid digits found for number"); + (token::Integer, sym::integer(0)) + } else { + self.validate_int_literal(base, start, suffix_start); + (token::Integer, self.symbol_from_to(start, suffix_start)) + } + }, + rustc_lexer::LiteralKind::Float { base, empty_exponent } => { + if empty_exponent { + let mut err = self.struct_span_fatal( + start, self.pos, + "expected at least one digit in exponent" + ); + err.emit(); + } + + match base { + Base::Hexadecimal => { + self.err_span_(start, suffix_start, + "hexadecimal float literal is not supported") + } + Base::Octal => { + self.err_span_(start, suffix_start, + "octal float literal is not supported") + } + Base::Binary => { + self.err_span_(start, suffix_start, + "binary float literal is not supported") + } + _ => () + } + + let id = self.symbol_from_to(start, suffix_start); + (token::Float, id) + }, + } } #[inline] fn src_index(&self, pos: BytePos) -> usize { - (pos - self.source_file.start_pos).to_usize() + (pos - self.start_pos).to_usize() } - /// Calls `f` with a string slice of the source text spanning from `start` - /// up to but excluding `self.pos`, meaning the slice does not include - /// the character `self.ch`. - fn with_str_from<T, F>(&self, start: BytePos, f: F) -> T - where F: FnOnce(&str) -> T + /// Slice of the source text from `start` up to but excluding `self.pos`, + /// meaning the slice does not include the character `self.ch`. + fn str_from(&self, start: BytePos) -> &str { - self.with_str_from_to(start, self.pos, f) + self.str_from_to(start, self.pos) } - /// Creates a Name from a given offset to the current offset, each - /// adjusted 1 towards each other (assumes that on either side there is a - /// single-byte delimiter). - fn name_from(&self, start: BytePos) -> ast::Name { + /// Creates a Symbol from a given offset to the current offset. + fn symbol_from(&self, start: BytePos) -> Symbol { debug!("taking an ident from {:?} to {:?}", start, self.pos); - self.with_str_from(start, Symbol::intern) + Symbol::intern(self.str_from(start)) } - /// As name_from, with an explicit endpoint. - fn name_from_to(&self, start: BytePos, end: BytePos) -> ast::Name { + /// As symbol_from, with an explicit endpoint. + fn symbol_from_to(&self, start: BytePos, end: BytePos) -> Symbol { debug!("taking an ident from {:?} to {:?}", start, end); - self.with_str_from_to(start, end, Symbol::intern) + Symbol::intern(self.str_from_to(start, end)) } - /// Calls `f` with a string slice of the source text spanning from `start` - /// up to but excluding `end`. - fn with_str_from_to<T, F>(&self, start: BytePos, end: BytePos, f: F) -> T - where F: FnOnce(&str) -> T + /// Slice of the source text spanning from `start` up to but excluding `end`. + fn str_from_to(&self, start: BytePos, end: BytePos) -> &str { - f(&self.src[self.src_index(start)..self.src_index(end)]) + &self.src[self.src_index(start)..self.src_index(end)] } /// Converts CRLF to LF in the given string, raising an error on bare CR. @@ -515,1360 +562,163 @@ impl<'a> StringReader<'a> { } } - /// Advance the StringReader by one character. - crate fn bump(&mut self) { - let next_src_index = self.src_index(self.next_pos); - if next_src_index < self.end_src_index { - let next_ch = char_at(&self.src, next_src_index); - let next_ch_len = next_ch.len_utf8(); - - self.ch = Some(next_ch); - self.pos = self.next_pos; - self.next_pos = self.next_pos + Pos::from_usize(next_ch_len); - } else { - self.ch = None; - self.pos = self.next_pos; - } - } - - fn nextch(&self) -> Option<char> { - let next_src_index = self.src_index(self.next_pos); - if next_src_index < self.end_src_index { - Some(char_at(&self.src, next_src_index)) - } else { - None - } - } - - #[inline] - fn nextch_is(&self, c: char) -> bool { - self.nextch() == Some(c) - } - - fn nextnextch(&self) -> Option<char> { - let next_src_index = self.src_index(self.next_pos); - if next_src_index < self.end_src_index { - let next_next_src_index = - next_src_index + char_at(&self.src, next_src_index).len_utf8(); - if next_next_src_index < self.end_src_index { - return Some(char_at(&self.src, next_next_src_index)); - } + fn report_non_started_raw_string(&self, start: BytePos) -> ! { + let bad_char = self.str_from(start).chars().last().unwrap(); + self + .struct_fatal_span_char( + start, + self.pos, + "found invalid character; only `#` is allowed \ + in raw string delimitation", + bad_char, + ) + .emit(); + FatalError.raise() + } + + fn report_unterminated_raw_string(&self, start: BytePos, n_hashes: usize) -> ! { + let mut err = self.struct_span_fatal( + start, start, + "unterminated raw string", + ); + err.span_label( + self.mk_sp(start, start), + "unterminated raw string", + ); + + if n_hashes > 0 { + err.note(&format!("this raw string should be terminated with `\"{}`", + "#".repeat(n_hashes as usize))); } - None - } - #[inline] - fn nextnextch_is(&self, c: char) -> bool { - self.nextnextch() == Some(c) + err.emit(); + FatalError.raise() } - /// Eats <XID_start><XID_continue>*, if possible. - fn scan_optional_raw_name(&mut self) -> Option<ast::Name> { - if !ident_start(self.ch) { - return None; - } - - let start = self.pos; - self.bump(); - - while ident_continue(self.ch) { - self.bump(); - } - - self.with_str_from(start, |string| { - if string == "_" { - self.sess.span_diagnostic - .struct_span_warn(self.mk_sp(start, self.pos), - "underscore literal suffix is not allowed") - .warn("this was previously accepted by the compiler but is \ - being phased out; it will become a hard error in \ - a future release!") - .note("for more information, see issue #42326 \ - <https://github.com/rust-lang/rust/issues/42326>") - .emit(); - None - } else { - Some(Symbol::intern(string)) + fn restrict_n_hashes(&self, start: BytePos, n_hashes: usize) -> u16 { + match n_hashes.try_into() { + Ok(n_hashes) => n_hashes, + Err(_) => { + self.fatal_span_(start, + self.pos, + "too many `#` symbols: raw strings may be \ + delimited by up to 65535 `#` symbols").raise(); + } + } + } + + fn validate_char_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + if let Err((off, err)) = unescape::unescape_char(lit) { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::Char, + 0..off, + err, + ) + } + } + + fn validate_byte_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + if let Err((off, err)) = unescape::unescape_byte(lit) { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::Byte, + 0..off, + err, + ) + } + } + + fn validate_str_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + unescape::unescape_str(lit, &mut |range, c| { + if let Err(err) = c { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::Str, + range, + err, + ) } }) } - /// PRECONDITION: self.ch is not whitespace - /// Eats any kind of comment. - fn scan_comment(&mut self) -> Option<TokenAndSpan> { - if let Some(c) = self.ch { - if c.is_whitespace() { - let msg = "called consume_any_line_comment, but there was whitespace"; - self.sess.span_diagnostic.span_err(self.mk_sp(self.pos, self.pos), msg); - } - } - - if self.ch_is('/') { - match self.nextch() { - Some('/') => { - self.bump(); - self.bump(); - - // line comments starting with "///" or "//!" are doc-comments - let doc_comment = (self.ch_is('/') && !self.nextch_is('/')) || self.ch_is('!'); - let start_bpos = self.pos - BytePos(2); - - while !self.is_eof() { - match self.ch.unwrap() { - '\n' => break, - '\r' => { - if self.nextch_is('\n') { - // CRLF - break; - } else if doc_comment { - self.err_span_(self.pos, - self.next_pos, - "bare CR not allowed in doc-comment"); - } - } - _ => (), - } - self.bump(); - } - - if doc_comment { - self.with_str_from(start_bpos, |string| { - // comments with only more "/"s are not doc comments - let tok = if is_doc_comment(string) { - token::DocComment(Symbol::intern(string)) - } else { - token::Comment - }; - - Some(TokenAndSpan { - tok, - sp: self.mk_sp(start_bpos, self.pos), - }) - }) - } else { - Some(TokenAndSpan { - tok: token::Comment, - sp: self.mk_sp(start_bpos, self.pos), - }) - } - } - Some('*') => { - self.bump(); - self.bump(); - self.scan_block_comment() - } - _ => None, - } - } else if self.ch_is('#') { - if self.nextch_is('!') { - - // Parse an inner attribute. - if self.nextnextch_is('[') { - return None; - } - - // I guess this is the only way to figure out if - // we're at the beginning of the file... - let smap = SourceMap::new(FilePathMapping::empty()); - smap.files.borrow_mut().source_files.push(self.source_file.clone()); - let loc = smap.lookup_char_pos_adj(self.pos); - debug!("Skipping a shebang"); - if loc.line == 1 && loc.col == CharPos(0) { - // FIXME: Add shebang "token", return it - let start = self.pos; - while !self.ch_is('\n') && !self.is_eof() { - self.bump(); - } - return Some(TokenAndSpan { - tok: token::Shebang(self.name_from(start)), - sp: self.mk_sp(start, self.pos), - }); - } + fn validate_raw_str_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + unescape::unescape_raw_str(lit, &mut |range, c| { + if let Err(err) = c { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::Str, + range, + err, + ) } - None - } else { - None - } - } - - /// If there is whitespace, shebang, or a comment, scan it. Otherwise, - /// return `None`. - fn scan_whitespace_or_comment(&mut self) -> Option<TokenAndSpan> { - match self.ch.unwrap_or('\0') { - // # to handle shebang at start of file -- this is the entry point - // for skipping over all "junk" - '/' | '#' => { - let c = self.scan_comment(); - debug!("scanning a comment {:?}", c); - c - }, - c if is_pattern_whitespace(Some(c)) => { - let start_bpos = self.pos; - while is_pattern_whitespace(self.ch) { - self.bump(); - } - let c = Some(TokenAndSpan { - tok: token::Whitespace, - sp: self.mk_sp(start_bpos, self.pos), - }); - debug!("scanning whitespace: {:?}", c); - c - } - _ => None, - } - } - - /// Might return a sugared-doc-attr - fn scan_block_comment(&mut self) -> Option<TokenAndSpan> { - // block comments starting with "/**" or "/*!" are doc-comments - let is_doc_comment = self.ch_is('*') || self.ch_is('!'); - let start_bpos = self.pos - BytePos(2); - - let mut level: isize = 1; - let mut has_cr = false; - while level > 0 { - if self.is_eof() { - let msg = if is_doc_comment { - "unterminated block doc-comment" - } else { - "unterminated block comment" - }; - let last_bpos = self.pos; - self.fatal_span_(start_bpos, last_bpos, msg).raise(); - } - let n = self.ch.unwrap(); - match n { - '/' if self.nextch_is('*') => { - level += 1; - self.bump(); - } - '*' if self.nextch_is('/') => { - level -= 1; - self.bump(); - } - '\r' => { - has_cr = true; - } - _ => (), - } - self.bump(); - } - - self.with_str_from(start_bpos, |string| { - // but comments with only "*"s between two "/"s are not - let tok = if is_block_doc_comment(string) { - let string = if has_cr { - self.translate_crlf(start_bpos, - string, - "bare CR not allowed in block doc-comment") - } else { - string.into() - }; - token::DocComment(Symbol::intern(&string[..])) - } else { - token::Comment - }; - - Some(TokenAndSpan { - tok, - sp: self.mk_sp(start_bpos, self.pos), - }) }) } - /// Scan through any digits (base `scan_radix`) or underscores, - /// and return how many digits there were. - /// - /// `real_radix` represents the true radix of the number we're - /// interested in, and errors will be emitted for any digits - /// between `real_radix` and `scan_radix`. - fn scan_digits(&mut self, real_radix: u32, scan_radix: u32) -> usize { - assert!(real_radix <= scan_radix); - let mut len = 0; - - loop { - let c = self.ch; - if c == Some('_') { - debug!("skipping a _"); - self.bump(); - continue; - } - match c.and_then(|cc| cc.to_digit(scan_radix)) { - Some(_) => { - debug!("{:?} in scan_digits", c); - // check that the hypothetical digit is actually - // in range for the true radix - if c.unwrap().to_digit(real_radix).is_none() { - self.err_span_(self.pos, - self.next_pos, - &format!("invalid digit for a base {} literal", real_radix)); - } - len += 1; - self.bump(); - } - _ => return len, - } - } - } - - /// Lex a LIT_INTEGER or a LIT_FLOAT - fn scan_number(&mut self, c: char) -> token::Lit { - let mut base = 10; - let start_bpos = self.pos; - self.bump(); - - let num_digits = if c == '0' { - match self.ch.unwrap_or('\0') { - 'b' => { - self.bump(); - base = 2; - self.scan_digits(2, 10) - } - 'o' => { - self.bump(); - base = 8; - self.scan_digits(8, 10) - } - 'x' => { - self.bump(); - base = 16; - self.scan_digits(16, 16) - } - '0'..='9' | '_' | '.' | 'e' | 'E' => { - self.scan_digits(10, 10) + 1 - } - _ => { - // just a 0 - return token::Integer(self.name_from(start_bpos)); - } - } - } else if c.is_digit(10) { - self.scan_digits(10, 10) + 1 - } else { - 0 - }; - - if num_digits == 0 { - self.err_span_(start_bpos, self.pos, "no valid digits found for number"); - - return token::Integer(Symbol::intern("0")); - } - - // might be a float, but don't be greedy if this is actually an - // integer literal followed by field/method access or a range pattern - // (`0..2` and `12.foo()`) - if self.ch_is('.') && !self.nextch_is('.') && - !ident_start(self.nextch()) { - // might have stuff after the ., and if it does, it needs to start - // with a number - self.bump(); - if self.ch.unwrap_or('\0').is_digit(10) { - self.scan_digits(10, 10); - self.scan_float_exponent(); - } - let pos = self.pos; - self.check_float_base(start_bpos, pos, base); - - token::Float(self.name_from(start_bpos)) - } else { - // it might be a float if it has an exponent - if self.ch_is('e') || self.ch_is('E') { - self.scan_float_exponent(); - let pos = self.pos; - self.check_float_base(start_bpos, pos, base); - return token::Float(self.name_from(start_bpos)); - } - // but we certainly have an integer! - token::Integer(self.name_from(start_bpos)) - } - } - - /// Scan over `n_digits` hex digits, stopping at `delim`, reporting an - /// error if too many or too few digits are encountered. - fn scan_hex_digits(&mut self, n_digits: usize, delim: char, below_0x7f_only: bool) -> bool { - debug!("scanning {} digits until {:?}", n_digits, delim); - let start_bpos = self.pos; - let mut accum_int = 0; - - let mut valid = true; - for _ in 0..n_digits { - if self.is_eof() { - let last_bpos = self.pos; - self.fatal_span_(start_bpos, - last_bpos, - "unterminated numeric character escape").raise(); - } - if self.ch_is(delim) { - let last_bpos = self.pos; - self.err_span_(start_bpos, - last_bpos, - "numeric character escape is too short"); - valid = false; - break; - } - let c = self.ch.unwrap_or('\x00'); - accum_int *= 16; - accum_int += c.to_digit(16).unwrap_or_else(|| { - self.err_span_char(self.pos, - self.next_pos, - "invalid character in numeric character escape", - c); - - valid = false; - 0 - }); - self.bump(); - } - - if below_0x7f_only && accum_int >= 0x80 { - self.err_span_(start_bpos, - self.pos, - "this form of character escape may only be used with characters in \ - the range [\\x00-\\x7f]"); - valid = false; - } - - match char::from_u32(accum_int) { - Some(_) => valid, - None => { - let last_bpos = self.pos; - self.err_span_(start_bpos, last_bpos, "invalid numeric character escape"); - false - } - } - } - - /// Scan for a single (possibly escaped) byte or char - /// in a byte, (non-raw) byte string, char, or (non-raw) string literal. - /// `start` is the position of `first_source_char`, which is already consumed. - /// - /// Returns `true` if there was a valid char/byte. - fn scan_char_or_byte(&mut self, - start: BytePos, - first_source_char: char, - ascii_only: bool, - delim: char) - -> bool - { - match first_source_char { - '\\' => { - // '\X' for some X must be a character constant: - let escaped = self.ch; - let escaped_pos = self.pos; - self.bump(); - match escaped { - None => {} // EOF here is an error that will be checked later. - Some(e) => { - return match e { - 'n' | 'r' | 't' | '\\' | '\'' | '"' | '0' => true, - 'x' => self.scan_byte_escape(delim, !ascii_only), - 'u' => { - let valid = if self.ch_is('{') { - self.scan_unicode_escape(delim) && !ascii_only - } else { - let span = self.mk_sp(start, self.pos); - let mut suggestion = "\\u{".to_owned(); - let mut err = self.sess.span_diagnostic.struct_span_err( - span, - "incorrect unicode escape sequence", - ); - let mut i = 0; - while let (Some(ch), true) = (self.ch, i < 6) { - if ch.is_digit(16) { - suggestion.push(ch); - self.bump(); - i += 1; - } else { - break; - } - } - if i != 0 { - suggestion.push('}'); - err.span_suggestion( - self.mk_sp(start, self.pos), - "format of unicode escape sequences uses braces", - suggestion, - Applicability::MaybeIncorrect, - ); - } else { - err.span_help( - span, - "format of unicode escape sequences is `\\u{...}`", - ); - } - err.emit(); - false - }; - if ascii_only { - self.err_span_(start, - self.pos, - "unicode escape sequences cannot be used as a \ - byte or in a byte string"); - } - valid - - } - '\n' if delim == '"' => { - self.consume_whitespace(); - true - } - '\r' if delim == '"' && self.ch_is('\n') => { - self.consume_whitespace(); - true - } - c => { - let pos = self.pos; - let mut err = self.struct_err_span_char(escaped_pos, - pos, - if ascii_only { - "unknown byte escape" - } else { - "unknown character \ - escape" - }, - c); - if e == '\r' { - err.span_help(self.mk_sp(escaped_pos, pos), - "this is an isolated carriage return; consider \ - checking your editor and version control \ - settings"); - } - if (e == '{' || e == '}') && !ascii_only { - err.span_help(self.mk_sp(escaped_pos, pos), - "if used in a formatting string, curly braces \ - are escaped with `{{` and `}}`"); - } - err.emit(); - false - } - } - } - } - } - '\t' | '\n' | '\r' | '\'' if delim == '\'' => { - let pos = self.pos; - self.err_span_char(start, - pos, - if ascii_only { - "byte constant must be escaped" - } else { - "character constant must be escaped" - }, - first_source_char); - return false; + fn validate_raw_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + unescape::unescape_raw_byte_str(lit, &mut |range, c| { + if let Err(err) = c { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::ByteStr, + range, + err, + ) } - '\r' => { - if self.ch_is('\n') { - self.bump(); - return true; - } else { - self.err_span_(start, - self.pos, - "bare CR not allowed in string, use \\r instead"); - return false; - } - } - _ => { - if ascii_only && first_source_char > '\x7F' { - let pos = self.pos; - self.err_span_(start, - pos, - "byte constant must be ASCII. Use a \\xHH escape for a \ - non-ASCII byte"); - return false; - } - } - } - true - } - - /// Scan over a `\u{...}` escape - /// - /// At this point, we have already seen the `\` and the `u`, the `{` is the current character. - /// We will read a hex number (with `_` separators), with 1 to 6 actual digits, - /// and pass over the `}`. - fn scan_unicode_escape(&mut self, delim: char) -> bool { - self.bump(); // past the { - let start_bpos = self.pos; - let mut valid = true; - - if let Some('_') = self.ch { - // disallow leading `_` - self.err_span_(self.pos, - self.next_pos, - "invalid start of unicode escape"); - valid = false; - } - - let count = self.scan_digits(16, 16); - - if count > 6 { - self.err_span_(start_bpos, - self.pos, - "overlong unicode escape (must have at most 6 hex digits)"); - valid = false; - } - - loop { - match self.ch { - Some('}') => { - if valid && count == 0 { - self.err_span_(start_bpos, - self.pos, - "empty unicode escape (must have at least 1 hex digit)"); - valid = false; - } - self.bump(); // past the ending `}` - break; - }, - Some(c) => { - if c == delim { - self.err_span_(self.pos, - self.pos, - "unterminated unicode escape (needed a `}`)"); - valid = false; - break; - } else if valid { - self.err_span_char(start_bpos, - self.pos, - "invalid character in unicode escape", - c); - valid = false; - } - }, - None => { - self.fatal_span_(start_bpos, - self.pos, - "unterminated unicode escape (found EOF)").raise(); - } - } - self.bump(); - } - - valid - } - - /// Scan over a float exponent. - fn scan_float_exponent(&mut self) { - if self.ch_is('e') || self.ch_is('E') { - self.bump(); - - if self.ch_is('-') || self.ch_is('+') { - self.bump(); - } - - if self.scan_digits(10, 10) == 0 { - let mut err = self.struct_span_fatal( - self.pos, self.next_pos, - "expected at least one digit in exponent" - ); - if let Some(ch) = self.ch { - // check for e.g., Unicode minus '−' (Issue #49746) - if unicode_chars::check_for_substitution(self, ch, &mut err) { - self.bump(); - self.scan_digits(10, 10); - } - } - err.emit(); - } - } - } - - /// Checks that a base is valid for a floating literal, emitting a nice - /// error if it isn't. - fn check_float_base(&mut self, start_bpos: BytePos, last_bpos: BytePos, base: usize) { - match base { - 16 => { - self.err_span_(start_bpos, - last_bpos, - "hexadecimal float literal is not supported") - } - 8 => { - self.err_span_(start_bpos, - last_bpos, - "octal float literal is not supported") - } - 2 => { - self.err_span_(start_bpos, - last_bpos, - "binary float literal is not supported") - } - _ => (), - } - } - - fn binop(&mut self, op: token::BinOpToken) -> token::Token { - self.bump(); - if self.ch_is('=') { - self.bump(); - token::BinOpEq(op) - } else { - token::BinOp(op) - } + }) } - /// Returns the next token from the string, advances the input past that - /// token, and updates the interner - fn next_token_inner(&mut self) -> Result<token::Token, ()> { - let c = self.ch; - - if ident_start(c) { - let (is_ident_start, is_raw_ident) = - match (c.unwrap(), self.nextch(), self.nextnextch()) { - // r# followed by an identifier starter is a raw identifier. - // This is an exception to the r# case below. - ('r', Some('#'), x) if ident_start(x) => (true, true), - // r as in r" or r#" is part of a raw string literal. - // b as in b' is part of a byte literal. - // They are not identifiers, and are handled further down. - ('r', Some('"'), _) | - ('r', Some('#'), _) | - ('b', Some('"'), _) | - ('b', Some('\''), _) | - ('b', Some('r'), Some('"')) | - ('b', Some('r'), Some('#')) => (false, false), - _ => (true, false), - }; - - if is_ident_start { - let raw_start = self.pos; - if is_raw_ident { - // Consume the 'r#' characters. - self.bump(); - self.bump(); - } - - let start = self.pos; - self.bump(); - - while ident_continue(self.ch) { - self.bump(); - } - - return Ok(self.with_str_from(start, |string| { - // FIXME: perform NFKC normalization here. (Issue #2253) - let ident = self.mk_ident(string); - - if is_raw_ident && (ident.is_path_segment_keyword() || - ident.name == keywords::Underscore.name()) { - self.fatal_span_(raw_start, self.pos, - &format!("`r#{}` is not currently supported.", ident.name) - ).raise(); - } - - if is_raw_ident { - let span = self.mk_sp(raw_start, self.pos); - self.sess.raw_identifier_spans.borrow_mut().push(span); - } - - token::Ident(ident, is_raw_ident) - })); - } - } - - if is_dec_digit(c) { - let num = self.scan_number(c.unwrap()); - let suffix = self.scan_optional_raw_name(); - debug!("next_token_inner: scanned number {:?}, {:?}", num, suffix); - return Ok(token::Literal(num, suffix)); - } - - match c.expect("next_token_inner called at EOF") { - // One-byte tokens. - ';' => { - self.bump(); - Ok(token::Semi) - } - ',' => { - self.bump(); - Ok(token::Comma) - } - '.' => { - self.bump(); - if self.ch_is('.') { - self.bump(); - if self.ch_is('.') { - self.bump(); - Ok(token::DotDotDot) - } else if self.ch_is('=') { - self.bump(); - Ok(token::DotDotEq) - } else { - Ok(token::DotDot) - } - } else { - Ok(token::Dot) - } - } - '(' => { - self.bump(); - Ok(token::OpenDelim(token::Paren)) - } - ')' => { - self.bump(); - Ok(token::CloseDelim(token::Paren)) - } - '{' => { - self.bump(); - Ok(token::OpenDelim(token::Brace)) - } - '}' => { - self.bump(); - Ok(token::CloseDelim(token::Brace)) - } - '[' => { - self.bump(); - Ok(token::OpenDelim(token::Bracket)) - } - ']' => { - self.bump(); - Ok(token::CloseDelim(token::Bracket)) - } - '@' => { - self.bump(); - Ok(token::At) - } - '#' => { - self.bump(); - Ok(token::Pound) - } - '~' => { - self.bump(); - Ok(token::Tilde) - } - '?' => { - self.bump(); - Ok(token::Question) - } - ':' => { - self.bump(); - if self.ch_is(':') { - self.bump(); - Ok(token::ModSep) - } else { - Ok(token::Colon) - } - } - - '$' => { - self.bump(); - Ok(token::Dollar) - } - - // Multi-byte tokens. - '=' => { - self.bump(); - if self.ch_is('=') { - self.bump(); - Ok(token::EqEq) - } else if self.ch_is('>') { - self.bump(); - Ok(token::FatArrow) - } else { - Ok(token::Eq) - } - } - '!' => { - self.bump(); - if self.ch_is('=') { - self.bump(); - Ok(token::Ne) - } else { - Ok(token::Not) - } - } - '<' => { - self.bump(); - match self.ch.unwrap_or('\x00') { - '=' => { - self.bump(); - Ok(token::Le) - } - '<' => { - Ok(self.binop(token::Shl)) - } - '-' => { - self.bump(); - Ok(token::LArrow) - } - _ => { - Ok(token::Lt) - } - } - } - '>' => { - self.bump(); - match self.ch.unwrap_or('\x00') { - '=' => { - self.bump(); - Ok(token::Ge) - } - '>' => { - Ok(self.binop(token::Shr)) - } - _ => { - Ok(token::Gt) - } - } - } - '\'' => { - // Either a character constant 'a' OR a lifetime name 'abc - let start_with_quote = self.pos; - self.bump(); - let start = self.pos; - - // the eof will be picked up by the final `'` check below - let c2 = self.ch.unwrap_or('\x00'); - self.bump(); - - // If the character is an ident start not followed by another single - // quote, then this is a lifetime name: - if (ident_start(Some(c2)) || c2.is_numeric()) && !self.ch_is('\'') { - while ident_continue(self.ch) { - self.bump(); - } - // lifetimes shouldn't end with a single quote - // if we find one, then this is an invalid character literal - if self.ch_is('\'') { - self.err_span_( - start_with_quote, - self.next_pos, - "character literal may only contain one codepoint"); - self.bump(); - return Ok(token::Literal(token::Err(Symbol::intern("??")), None)) - - } - - // Include the leading `'` in the real identifier, for macro - // expansion purposes. See #12512 for the gory details of why - // this is necessary. - let ident = self.with_str_from(start, |lifetime_name| { - self.mk_ident(&format!("'{}", lifetime_name)) - }); - - if c2.is_numeric() { - // this is a recovered lifetime written `'1`, error but accept it - self.err_span_( - start_with_quote, - self.pos, - "lifetimes cannot start with a number", - ); - } - - return Ok(token::Lifetime(ident)); - } - - let valid = self.scan_char_or_byte(start, c2, /* ascii_only */ false, '\''); - - if !self.ch_is('\'') { - let pos = self.pos; - - loop { - self.bump(); - if self.ch_is('\'') { - let start = self.src_index(start); - let end = self.src_index(self.pos); - self.bump(); - let span = self.mk_sp(start_with_quote, self.pos); - self.sess.span_diagnostic - .struct_span_err(span, - "character literal may only contain one codepoint") - .span_suggestion( - span, - "if you meant to write a `str` literal, use double quotes", - format!("\"{}\"", &self.src[start..end]), - Applicability::MachineApplicable - ).emit(); - return Ok(token::Literal(token::Err(Symbol::intern("??")), None)) - } - if self.ch_is('\n') || self.is_eof() || self.ch_is('/') { - // Only attempt to infer single line string literals. If we encounter - // a slash, bail out in order to avoid nonsensical suggestion when - // involving comments. - break; - } - } - - self.fatal_span_verbose(start_with_quote, pos, - String::from("character literal may only contain one codepoint")).raise(); - } - - let id = if valid { - self.name_from(start) - } else { - Symbol::intern("0") - }; - - self.bump(); // advance ch past token - let suffix = self.scan_optional_raw_name(); - - Ok(token::Literal(token::Char(id), suffix)) - } - 'b' => { - self.bump(); - let lit = match self.ch { - Some('\'') => self.scan_byte(), - Some('"') => self.scan_byte_string(), - Some('r') => self.scan_raw_byte_string(), - _ => unreachable!(), // Should have been a token::Ident above. - }; - let suffix = self.scan_optional_raw_name(); - - Ok(token::Literal(lit, suffix)) - } - '"' => { - let start_bpos = self.pos; - let mut valid = true; - self.bump(); - - while !self.ch_is('"') { - if self.is_eof() { - let last_bpos = self.pos; - self.fatal_span_(start_bpos, - last_bpos, - "unterminated double quote string").raise(); - } - - let ch_start = self.pos; - let ch = self.ch.unwrap(); - self.bump(); - valid &= self.scan_char_or_byte(ch_start, ch, /* ascii_only */ false, '"'); - } - // adjust for the ASCII " at the start of the literal - let id = if valid { - self.name_from(start_bpos + BytePos(1)) - } else { - Symbol::intern("??") - }; - self.bump(); - let suffix = self.scan_optional_raw_name(); - - Ok(token::Literal(token::Str_(id), suffix)) - } - 'r' => { - let start_bpos = self.pos; - self.bump(); - let mut hash_count: u16 = 0; - while self.ch_is('#') { - if hash_count == 65535 { - let bpos = self.next_pos; - self.fatal_span_(start_bpos, - bpos, - "too many `#` symbols: raw strings may be \ - delimited by up to 65535 `#` symbols").raise(); - } - self.bump(); - hash_count += 1; - } - - if self.is_eof() { - self.fail_unterminated_raw_string(start_bpos, hash_count); - } else if !self.ch_is('"') { - let last_bpos = self.pos; - let curr_char = self.ch.unwrap(); - self.fatal_span_char(start_bpos, - last_bpos, - "found invalid character; only `#` is allowed \ - in raw string delimitation", - curr_char).raise(); - } - self.bump(); - let content_start_bpos = self.pos; - let mut content_end_bpos; - let mut valid = true; - 'outer: loop { - if self.is_eof() { - self.fail_unterminated_raw_string(start_bpos, hash_count); - } - // if self.ch_is('"') { - // content_end_bpos = self.pos; - // for _ in 0..hash_count { - // self.bump(); - // if !self.ch_is('#') { - // continue 'outer; - let c = self.ch.unwrap(); - match c { - '"' => { - content_end_bpos = self.pos; - for _ in 0..hash_count { - self.bump(); - if !self.ch_is('#') { - continue 'outer; - } - } - break; - } - '\r' => { - if !self.nextch_is('\n') { - let last_bpos = self.pos; - self.err_span_(start_bpos, - last_bpos, - "bare CR not allowed in raw string, use \\r \ - instead"); - valid = false; - } - } - _ => (), - } - self.bump(); - } - - self.bump(); - let id = if valid { - self.name_from_to(content_start_bpos, content_end_bpos) - } else { - Symbol::intern("??") - }; - let suffix = self.scan_optional_raw_name(); - - Ok(token::Literal(token::StrRaw(id, hash_count), suffix)) - } - '-' => { - if self.nextch_is('>') { - self.bump(); - self.bump(); - Ok(token::RArrow) - } else { - Ok(self.binop(token::Minus)) - } - } - '&' => { - if self.nextch_is('&') { - self.bump(); - self.bump(); - Ok(token::AndAnd) - } else { - Ok(self.binop(token::And)) - } - } - '|' => { - match self.nextch() { - Some('|') => { - self.bump(); - self.bump(); - Ok(token::OrOr) - } - _ => { - Ok(self.binop(token::Or)) - } - } - } - '+' => { - Ok(self.binop(token::Plus)) - } - '*' => { - Ok(self.binop(token::Star)) - } - '/' => { - Ok(self.binop(token::Slash)) - } - '^' => { - Ok(self.binop(token::Caret)) - } - '%' => { - Ok(self.binop(token::Percent)) - } - c => { - let last_bpos = self.pos; - let bpos = self.next_pos; - let mut err = self.struct_fatal_span_char(last_bpos, - bpos, - "unknown start of token", - c); - unicode_chars::check_for_substitution(self, c, &mut err); - self.fatal_errs.push(err); - - Err(()) + fn validate_byte_str_escape(&self, content_start: BytePos, content_end: BytePos) { + let lit = self.str_from_to(content_start, content_end); + unescape::unescape_byte_str(lit, &mut |range, c| { + if let Err(err) = c { + emit_unescape_error( + &self.sess.span_diagnostic, + lit, + self.mk_sp(content_start - BytePos(1), content_end + BytePos(1)), + unescape::Mode::ByteStr, + range, + err, + ) } - } - } - - fn consume_whitespace(&mut self) { - while is_pattern_whitespace(self.ch) && !self.is_eof() { - self.bump(); - } - } - - fn read_to_eol(&mut self) -> String { - let mut val = String::new(); - while !self.ch_is('\n') && !self.is_eof() { - val.push(self.ch.unwrap()); - self.bump(); - } - - if self.ch_is('\n') { - self.bump(); - } - - val - } - - fn read_one_line_comment(&mut self) -> String { - let val = self.read_to_eol(); - assert!((val.as_bytes()[0] == b'/' && val.as_bytes()[1] == b'/') || - (val.as_bytes()[0] == b'#' && val.as_bytes()[1] == b'!')); - val - } - - fn consume_non_eol_whitespace(&mut self) { - while is_pattern_whitespace(self.ch) && !self.ch_is('\n') && !self.is_eof() { - self.bump(); - } - } - - fn peeking_at_comment(&self) -> bool { - (self.ch_is('/') && self.nextch_is('/')) || (self.ch_is('/') && self.nextch_is('*')) || - // consider shebangs comments, but not inner attributes - (self.ch_is('#') && self.nextch_is('!') && !self.nextnextch_is('[')) - } - - fn scan_byte(&mut self) -> token::Lit { - self.bump(); - let start = self.pos; - - // the eof will be picked up by the final `'` check below - let c2 = self.ch.unwrap_or('\x00'); - self.bump(); - - let valid = self.scan_char_or_byte(start, - c2, - // ascii_only = - true, - '\''); - if !self.ch_is('\'') { - // Byte offsetting here is okay because the - // character before position `start` are an - // ascii single quote and ascii 'b'. - let pos = self.pos; - self.fatal_span_verbose(start - BytePos(2), - pos, - "unterminated byte constant".to_string()).raise(); - } - - let id = if valid { - self.name_from(start) - } else { - Symbol::intern("?") - }; - self.bump(); // advance ch past token - - token::Byte(id) - } - - #[inline] - fn scan_byte_escape(&mut self, delim: char, below_0x7f_only: bool) -> bool { - self.scan_hex_digits(2, delim, below_0x7f_only) + }) } - fn scan_byte_string(&mut self) -> token::Lit { - self.bump(); - let start = self.pos; - let mut valid = true; - - while !self.ch_is('"') { - if self.is_eof() { - let pos = self.pos; - self.fatal_span_(start, pos, "unterminated double quote byte string").raise(); - } - - let ch_start = self.pos; - let ch = self.ch.unwrap(); - self.bump(); - valid &= self.scan_char_or_byte(ch_start, - ch, - // ascii_only = - true, - '"'); - } - - let id = if valid { - self.name_from(start) - } else { - Symbol::intern("??") + fn validate_int_literal(&self, base: Base, content_start: BytePos, content_end: BytePos) { + let base = match base { + Base::Binary => 2, + Base::Octal => 8, + _ => return, }; - self.bump(); - - token::ByteStr(id) - } + let s = self.str_from_to(content_start + BytePos(2), content_end); + for (idx, c) in s.char_indices() { + let idx = idx as u32; + if c != '_' && c.to_digit(base).is_none() { + let lo = content_start + BytePos(2 + idx); + let hi = content_start + BytePos(2 + idx + c.len_utf8() as u32); + self.err_span_(lo, hi, + &format!("invalid digit for a base {} literal", base)); - fn scan_raw_byte_string(&mut self) -> token::Lit { - let start_bpos = self.pos; - self.bump(); - let mut hash_count = 0; - while self.ch_is('#') { - if hash_count == 65535 { - let bpos = self.next_pos; - self.fatal_span_(start_bpos, - bpos, - "too many `#` symbols: raw byte strings may be \ - delimited by up to 65535 `#` symbols").raise(); } - self.bump(); - hash_count += 1; - } - - if self.is_eof() { - self.fail_unterminated_raw_string(start_bpos, hash_count); - } else if !self.ch_is('"') { - let pos = self.pos; - let ch = self.ch.unwrap(); - self.fatal_span_char(start_bpos, - pos, - "found invalid character; only `#` is allowed in raw \ - string delimitation", - ch).raise(); } - self.bump(); - let content_start_bpos = self.pos; - let mut content_end_bpos; - 'outer: loop { - match self.ch { - None => { - self.fail_unterminated_raw_string(start_bpos, hash_count); - } - Some('"') => { - content_end_bpos = self.pos; - for _ in 0..hash_count { - self.bump(); - if !self.ch_is('#') { - continue 'outer; - } - } - break; - } - Some(c) => { - if c > '\x7F' { - let pos = self.pos; - self.err_span_char(pos, pos, "raw byte string must be ASCII", c); - } - } - } - self.bump(); - } - - self.bump(); - - token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos), hash_count) } } -// This tests the character for the unicode property 'PATTERN_WHITE_SPACE' which -// is guaranteed to be forward compatible. http://unicode.org/reports/tr31/#R3 -#[inline] -crate fn is_pattern_whitespace(c: Option<char>) -> bool { - c.map_or(false, Pattern_White_Space) -} - -#[inline] -fn in_range(c: Option<char>, lo: char, hi: char) -> bool { - c.map_or(false, |c| lo <= c && c <= hi) -} - -#[inline] -fn is_dec_digit(c: Option<char>) -> bool { - in_range(c, '0', '9') -} - fn is_doc_comment(s: &str) -> bool { let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') || s.starts_with("//!"); @@ -1883,289 +733,3 @@ fn is_block_doc_comment(s: &str) -> bool { debug!("is {:?} a doc comment? {}", s, res); res } - -/// Determine whether `c` is a valid start for an ident. -fn ident_start(c: Option<char>) -> bool { - let c = match c { - Some(c) => c, - None => return false, - }; - - (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c > '\x7f' && c.is_xid_start()) -} - -fn ident_continue(c: Option<char>) -> bool { - let c = match c { - Some(c) => c, - None => return false, - }; - - (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || - (c > '\x7f' && c.is_xid_continue()) -} - -#[inline] -fn char_at(s: &str, byte: usize) -> char { - s[byte..].chars().next().unwrap() -} - -#[cfg(test)] -mod tests { - use super::*; - - use crate::ast::{Ident, CrateConfig}; - use crate::symbol::Symbol; - use crate::source_map::SourceMap; - use crate::feature_gate::UnstableFeatures; - use crate::parse::token; - use crate::diagnostics::plugin::ErrorMap; - use crate::with_globals; - use std::io; - use std::path::PathBuf; - use syntax_pos::{BytePos, Span, NO_EXPANSION}; - use rustc_data_structures::fx::FxHashSet; - use rustc_data_structures::sync::Lock; - - fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { - let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), - Some(sm.clone()), - false, - false); - ParseSess { - span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)), - unstable_features: UnstableFeatures::from_environment(), - config: CrateConfig::default(), - included_mod_stack: Lock::new(Vec::new()), - source_map: sm, - missing_fragment_specifiers: Lock::new(FxHashSet::default()), - raw_identifier_spans: Lock::new(Vec::new()), - registered_diagnostics: Lock::new(ErrorMap::new()), - buffered_lints: Lock::new(vec![]), - } - } - - // open a string reader for the given string - fn setup<'a>(sm: &SourceMap, - sess: &'a ParseSess, - teststr: String) - -> StringReader<'a> { - let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); - StringReader::new(sess, sf, None) - } - - #[test] - fn t1() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut string_reader = setup(&sm, - &sh, - "/* my source file */ fn main() { println!(\"zebra\"); }\n" - .to_string()); - let id = Ident::from_str("fn"); - assert_eq!(string_reader.next_token().tok, token::Comment); - assert_eq!(string_reader.next_token().tok, token::Whitespace); - let tok1 = string_reader.next_token(); - let tok2 = TokenAndSpan { - tok: token::Ident(id, false), - sp: Span::new(BytePos(21), BytePos(23), NO_EXPANSION), - }; - assert_eq!(tok1.tok, tok2.tok); - assert_eq!(tok1.sp, tok2.sp); - assert_eq!(string_reader.next_token().tok, token::Whitespace); - // the 'main' id is already read: - assert_eq!(string_reader.pos.clone(), BytePos(28)); - // read another token: - let tok3 = string_reader.next_token(); - let tok4 = TokenAndSpan { - tok: mk_ident("main"), - sp: Span::new(BytePos(24), BytePos(28), NO_EXPANSION), - }; - assert_eq!(tok3.tok, tok4.tok); - assert_eq!(tok3.sp, tok4.sp); - // the lparen is already read: - assert_eq!(string_reader.pos.clone(), BytePos(29)) - }) - } - - // check that the given reader produces the desired stream - // of tokens (stop checking after exhausting the expected vec) - fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<token::Token>) { - for expected_tok in &expected { - assert_eq!(&string_reader.next_token().tok, expected_tok); - } - } - - // make the identifier by looking up the string in the interner - fn mk_ident(id: &str) -> token::Token { - token::Token::from_ast_ident(Ident::from_str(id)) - } - - #[test] - fn doublecolonparsing() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a b".to_string()), - vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_2() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a::b".to_string()), - vec![mk_ident("a"), token::ModSep, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_3() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a ::b".to_string()), - vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_4() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a:: b".to_string()), - vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); - }) - } - - #[test] - fn character_a() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token().tok, - token::Literal(token::Char(Symbol::intern("a")), None)); - }) - } - - #[test] - fn character_space() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token().tok, - token::Literal(token::Char(Symbol::intern(" ")), None)); - }) - } - - #[test] - fn character_escaped() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token().tok, - token::Literal(token::Char(Symbol::intern("\\n")), None)); - }) - } - - #[test] - fn lifetime_name() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token().tok, - token::Lifetime(Ident::from_str("'abc"))); - }) - } - - #[test] - fn raw_string() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()) - .next_token() - .tok, - token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None)); - }) - } - - #[test] - fn literal_suffixes() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - macro_rules! test { - ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ - assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token().tok, - token::Literal(token::$tok_type(Symbol::intern($tok_contents)), - Some(Symbol::intern("suffix")))); - // with a whitespace separator: - assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token().tok, - token::Literal(token::$tok_type(Symbol::intern($tok_contents)), - None)); - }} - } - - test!("'a'", Char, "a"); - test!("b'a'", Byte, "a"); - test!("\"a\"", Str_, "a"); - test!("b\"a\"", ByteStr, "a"); - test!("1234", Integer, "1234"); - test!("0b101", Integer, "0b101"); - test!("0xABC", Integer, "0xABC"); - test!("1.0", Float, "1.0"); - test!("1.0e10", Float, "1.0e10"); - - assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token().tok, - token::Literal(token::Integer(Symbol::intern("2")), - Some(Symbol::intern("us")))); - assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::StrRaw(Symbol::intern("raw"), 3), - Some(Symbol::intern("suffix")))); - assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok, - token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3), - Some(Symbol::intern("suffix")))); - }) - } - - #[test] - fn line_doc_comments() { - assert!(is_doc_comment("///")); - assert!(is_doc_comment("/// blah")); - assert!(!is_doc_comment("////")); - } - - #[test] - fn nested_block_comments() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); - match lexer.next_token().tok { - token::Comment => {} - _ => panic!("expected a comment!"), - } - assert_eq!(lexer.next_token().tok, - token::Literal(token::Char(Symbol::intern("a")), None)); - }) - } - - #[test] - fn crlf_comments() { - with_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); - let comment = lexer.next_token(); - assert_eq!(comment.tok, token::Comment); - assert_eq!((comment.sp.lo(), comment.sp.hi()), (BytePos(0), BytePos(7))); - assert_eq!(lexer.next_token().tok, token::Whitespace); - assert_eq!(lexer.next_token().tok, - token::DocComment(Symbol::intern("/// test"))); - }) - } -} diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs new file mode 100644 index 00000000000..fc47e4f0b18 --- /dev/null +++ b/src/libsyntax/parse/lexer/tests.rs @@ -0,0 +1,255 @@ +use super::*; + +use crate::ast::CrateConfig; +use crate::symbol::Symbol; +use crate::source_map::{SourceMap, FilePathMapping}; +use crate::feature_gate::UnstableFeatures; +use crate::parse::token; +use crate::diagnostics::plugin::ErrorMap; +use crate::with_default_globals; +use std::io; +use std::path::PathBuf; +use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition}; +use rustc_data_structures::fx::{FxHashSet, FxHashMap}; +use rustc_data_structures::sync::{Lock, Once}; + +fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { + let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), + Some(sm.clone()), + false, + false, + false); + ParseSess { + span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)), + unstable_features: UnstableFeatures::from_environment(), + config: CrateConfig::default(), + included_mod_stack: Lock::new(Vec::new()), + source_map: sm, + missing_fragment_specifiers: Lock::new(FxHashSet::default()), + raw_identifier_spans: Lock::new(Vec::new()), + registered_diagnostics: Lock::new(ErrorMap::new()), + buffered_lints: Lock::new(vec![]), + edition: Edition::from_session(), + ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), + param_attr_spans: Lock::new(Vec::new()), + let_chains_spans: Lock::new(Vec::new()), + async_closure_spans: Lock::new(Vec::new()), + injected_crate_name: Once::new(), + } +} + +// open a string reader for the given string +fn setup<'a>(sm: &SourceMap, + sess: &'a ParseSess, + teststr: String) + -> StringReader<'a> { + let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); + StringReader::new(sess, sf, None) +} + +#[test] +fn t1() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut string_reader = setup(&sm, + &sh, + "/* my source file */ fn main() { println!(\"zebra\"); }\n" + .to_string()); + assert_eq!(string_reader.next_token(), token::Comment); + assert_eq!(string_reader.next_token(), token::Whitespace); + let tok1 = string_reader.next_token(); + let tok2 = Token::new( + mk_ident("fn"), + Span::new(BytePos(21), BytePos(23), NO_EXPANSION), + ); + assert_eq!(tok1.kind, tok2.kind); + assert_eq!(tok1.span, tok2.span); + assert_eq!(string_reader.next_token(), token::Whitespace); + // read another token: + let tok3 = string_reader.next_token(); + assert_eq!(string_reader.pos.clone(), BytePos(28)); + let tok4 = Token::new( + mk_ident("main"), + Span::new(BytePos(24), BytePos(28), NO_EXPANSION), + ); + assert_eq!(tok3.kind, tok4.kind); + assert_eq!(tok3.span, tok4.span); + + assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); + assert_eq!(string_reader.pos.clone(), BytePos(29)) + }) +} + +// check that the given reader produces the desired stream +// of tokens (stop checking after exhausting the expected vec) +fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) { + for expected_tok in &expected { + assert_eq!(&string_reader.next_token(), expected_tok); + } +} + +// make the identifier by looking up the string in the interner +fn mk_ident(id: &str) -> TokenKind { + token::Ident(Symbol::intern(id), false) +} + +fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { + TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) +} + +#[test] +fn doublecolonparsing() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a b".to_string()), + vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_2() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a::b".to_string()), + vec![mk_ident("a"), token::ModSep, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_3() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a ::b".to_string()), + vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_4() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a:: b".to_string()), + vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); + }) +} + +#[test] +fn character_a() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(), + mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn character_space() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(), + mk_lit(token::Char, " ", None)); + }) +} + +#[test] +fn character_escaped() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token(), + mk_lit(token::Char, "\\n", None)); + }) +} + +#[test] +fn lifetime_name() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token(), + token::Lifetime(Symbol::intern("'abc"))); + }) +} + +#[test] +fn raw_string() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), + mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None)); + }) +} + +#[test] +fn literal_suffixes() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + macro_rules! test { + ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ + assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, Some("suffix"))); + // with a whitespace separator: + assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, None)); + }} + } + + test!("'a'", Char, "a"); + test!("b'a'", Byte, "a"); + test!("\"a\"", Str, "a"); + test!("b\"a\"", ByteStr, "a"); + test!("1234", Integer, "1234"); + test!("0b101", Integer, "0b101"); + test!("0xABC", Integer, "0xABC"); + test!("1.0", Float, "1.0"); + test!("1.0e10", Float, "1.0e10"); + + assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token(), + mk_lit(token::Integer, "2", Some("us"))); + assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::StrRaw(3), "raw", Some("suffix"))); + assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::ByteStrRaw(3), "raw", Some("suffix"))); + }) +} + +#[test] +fn line_doc_comments() { + assert!(is_doc_comment("///")); + assert!(is_doc_comment("/// blah")); + assert!(!is_doc_comment("////")); +} + +#[test] +fn nested_block_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); + assert_eq!(lexer.next_token(), token::Comment); + assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn crlf_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); + let comment = lexer.next_token(); + assert_eq!(comment.kind, token::Comment); + assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); + assert_eq!(lexer.next_token(), token::Whitespace); + assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); + }) +} diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 0db36c84cdf..37e67a2729e 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -1,13 +1,47 @@ +use syntax_pos::Span; + use crate::print::pprust::token_to_string; use crate::parse::lexer::{StringReader, UnmatchedBrace}; -use crate::parse::{token, PResult}; -use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint}; +use crate::parse::token::{self, Token}; +use crate::parse::PResult; +use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { + crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) { + let mut tt_reader = TokenTreesReader { + string_reader: self, + token: Token::dummy(), + joint_to_prev: Joint, + open_braces: Vec::new(), + unmatched_braces: Vec::new(), + matching_delim_spans: Vec::new(), + last_unclosed_found_span: None, + }; + let res = tt_reader.parse_all_token_trees(); + (res, tt_reader.unmatched_braces) + } +} + +struct TokenTreesReader<'a> { + string_reader: StringReader<'a>, + token: Token, + joint_to_prev: IsJoint, + /// Stack of open delimiters and their spans. Used for error message. + open_braces: Vec<(token::DelimToken, Span)>, + unmatched_braces: Vec<UnmatchedBrace>, + /// The type and spans for all braces + /// + /// Used only for error recovery when arriving to EOF with mismatched braces. + matching_delim_spans: Vec<(token::DelimToken, Span, Span)>, + last_unclosed_found_span: Option<Span>, +} + +impl<'a> TokenTreesReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. - crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> { + fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> { let mut tts = Vec::new(); + self.real_token(); while self.token != token::Eof { tts.push(self.parse_token_tree()?); } @@ -19,7 +53,7 @@ impl<'a> StringReader<'a> { fn parse_token_trees_until_close_delim(&mut self) -> TokenStream { let mut tts = vec![]; loop { - if let token::CloseDelim(..) = self.token { + if let token::CloseDelim(..) = self.token.kind { return TokenStream::new(tts); } @@ -34,11 +68,12 @@ impl<'a> StringReader<'a> { } fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> { - let sm = self.sess.source_map(); - match self.token { + let sm = self.string_reader.sess.source_map(); + match self.token.kind { token::Eof => { let msg = "this file contains an un-closed delimiter"; - let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg); + let mut err = self.string_reader.sess.span_diagnostic + .struct_span_err(self.token.span, msg); for &(_, sp) in &self.open_braces { err.span_label(sp, "un-closed delimiter"); } @@ -46,13 +81,12 @@ impl<'a> StringReader<'a> { if let Some((delim, _)) = self.open_braces.last() { if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter() .filter(|(d, open_sp, close_sp)| { - - if let Some(close_padding) = sm.span_to_margin(*close_sp) { - if let Some(open_padding) = sm.span_to_margin(*open_sp) { - return delim == d && close_padding != open_padding; + if let Some(close_padding) = sm.span_to_margin(*close_sp) { + if let Some(open_padding) = sm.span_to_margin(*open_sp) { + return delim == d && close_padding != open_padding; + } } - } - false + false }).next() // these are in reverse order as they get inserted on close, but { // we want the last open/first close err.span_label( @@ -69,10 +103,10 @@ impl<'a> StringReader<'a> { }, token::OpenDelim(delim) => { // The span for beginning of the delimited section - let pre_span = self.span; + let pre_span = self.token.span; // Parse the open delimiter. - self.open_braces.push((delim, self.span)); + self.open_braces.push((delim, self.token.span)); self.real_token(); // Parse the token trees within the delimiters. @@ -81,9 +115,9 @@ impl<'a> StringReader<'a> { let tts = self.parse_token_trees_until_close_delim(); // Expand to cover the entire delimited token tree - let delim_span = DelimSpan::from_pair(pre_span, self.span); + let delim_span = DelimSpan::from_pair(pre_span, self.token.span); - match self.token { + match self.token.kind { // Correct delimiter. token::CloseDelim(d) if d == delim => { let (open_brace, open_brace_span) = self.open_braces.pop().unwrap(); @@ -93,7 +127,7 @@ impl<'a> StringReader<'a> { self.matching_delim_spans.clear(); } else { self.matching_delim_spans.push( - (open_brace, open_brace_span, self.span), + (open_brace, open_brace_span, self.token.span), ); } // Parse the close delimiter. @@ -103,16 +137,16 @@ impl<'a> StringReader<'a> { token::CloseDelim(other) => { let mut unclosed_delimiter = None; let mut candidate = None; - if self.last_unclosed_found_span != Some(self.span) { + if self.last_unclosed_found_span != Some(self.token.span) { // do not complain about the same unclosed delimiter multiple times - self.last_unclosed_found_span = Some(self.span); + self.last_unclosed_found_span = Some(self.token.span); // This is a conservative error: only report the last unclosed // delimiter. The previous unclosed delimiters could actually be // closed! The parser just hasn't gotten to them yet. if let Some(&(_, sp)) = self.open_braces.last() { unclosed_delimiter = Some(sp); }; - if let Some(current_padding) = sm.span_to_margin(self.span) { + if let Some(current_padding) = sm.span_to_margin(self.token.span) { for (brace, brace_span) in &self.open_braces { if let Some(padding) = sm.span_to_margin(*brace_span) { // high likelihood of these two corresponding @@ -126,7 +160,7 @@ impl<'a> StringReader<'a> { self.unmatched_braces.push(UnmatchedBrace { expected_delim: tok, found_delim: other, - found_span: self.span, + found_span: self.token.span, unclosed_span: unclosed_delimiter, candidate_span: candidate, }); @@ -164,20 +198,33 @@ impl<'a> StringReader<'a> { // matching opening delimiter). let token_str = token_to_string(&self.token); let msg = format!("unexpected close delimiter: `{}`", token_str); - let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg); - err.span_label(self.span, "unexpected close delimiter"); + let mut err = self.string_reader.sess.span_diagnostic + .struct_span_err(self.token.span, &msg); + err.span_label(self.token.span, "unexpected close delimiter"); Err(err) }, _ => { - let tt = TokenTree::Token(self.span, self.token.clone()); - // Note that testing for joint-ness here is done via the raw - // source span as the joint-ness is a property of the raw source - // rather than wanting to take `override_span` into account. - let raw = self.span_src_raw; + let tt = TokenTree::Token(self.token.take()); self.real_token(); - let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token); + let is_joint = self.joint_to_prev == Joint && self.token.is_op(); Ok((tt, if is_joint { Joint } else { NonJoint })) } } } + + fn real_token(&mut self) { + self.joint_to_prev = Joint; + loop { + let token = self.string_reader.next_token(); + match token.kind { + token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => { + self.joint_to_prev = NonJoint; + } + _ => { + self.token = token; + return; + }, + } + } + } } diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 94ce6297fbe..eaa736c6a35 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -1,10 +1,12 @@ // Characters and their corresponding confusables were collected from // http://www.unicode.org/Public/security/10.0.0/confusables.txt -use syntax_pos::{Span, Pos, NO_EXPANSION}; -use errors::{Applicability, DiagnosticBuilder}; use super::StringReader; +use errors::{Applicability, DiagnosticBuilder}; +use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION, symbol::kw}; +use crate::parse::token; +#[rustfmt::skip] // for line breaks const UNICODE_ARRAY: &[(char, &str, char)] = &[ (' ', "Line Separator", ' '), (' ', "Paragraph Separator", ' '), @@ -49,7 +51,7 @@ const UNICODE_ARRAY: &[(char, &str, char)] = &[ ('─', "Box Drawings Light Horizontal", '-'), ('━', "Box Drawings Heavy Horizontal", '-'), ('㇐', "CJK Stroke H", '-'), - ('ꟷ', "Latin Epigraphic Letter Dideways", '-'), + ('ꟷ', "Latin Epigraphic Letter Sideways I", '-'), ('ᅳ', "Hangul Jungseong Eu", '-'), ('ㅡ', "Hangul Letter Eu", '-'), ('一', "CJK Unified Ideograph-4E00", '-'), @@ -293,74 +295,99 @@ const UNICODE_ARRAY: &[(char, &str, char)] = &[ ('〉', "Right-Pointing Angle Bracket", '>'), ('〉', "Right Angle Bracket", '>'), ('》', "Right Double Angle Bracket", '>'), - ('＞', "Fullwidth Greater-Than Sign", '>'), ]; - - -const ASCII_ARRAY: &[(char, &str)] = &[ - (' ', "Space"), - ('_', "Underscore"), - ('-', "Minus/Hyphen"), - (',', "Comma"), - (';', "Semicolon"), - (':', "Colon"), - ('!', "Exclamation Mark"), - ('?', "Question Mark"), - ('.', "Period"), - ('\'', "Single Quote"), - ('"', "Quotation Mark"), - ('(', "Left Parenthesis"), - (')', "Right Parenthesis"), - ('[', "Left Square Bracket"), - (']', "Right Square Bracket"), - ('{', "Left Curly Brace"), - ('}', "Right Curly Brace"), - ('*', "Asterisk"), - ('/', "Slash"), - ('\\', "Backslash"), - ('&', "Ampersand"), - ('+', "Plus Sign"), - ('<', "Less-Than Sign"), - ('=', "Equals Sign"), - ('>', "Greater-Than Sign"), ]; - -crate fn check_for_substitution<'a>(reader: &StringReader<'a>, - ch: char, - err: &mut DiagnosticBuilder<'a>) -> bool { - UNICODE_ARRAY - .iter() - .find(|&&(c, _, _)| c == ch) - .map(|&(_, u_name, ascii_char)| { - let span = Span::new(reader.pos, reader.next_pos, NO_EXPANSION); - match ASCII_ARRAY.iter().find(|&&(c, _)| c == ascii_char) { - Some(&(ascii_char, ascii_name)) => { - // special help suggestion for "directed" double quotes - if let Some(s) = reader.peek_delimited('“', '”') { - let msg = format!("Unicode characters '“' (Left Double Quotation Mark) and \ - '”' (Right Double Quotation Mark) look like '{}' ({}), but are not", - ascii_char, ascii_name); - err.span_suggestion( - Span::new(reader.pos, reader.next_pos + Pos::from_usize(s.len()) + - Pos::from_usize('”'.len_utf8()), NO_EXPANSION), - &msg, - format!("\"{}\"", s), - Applicability::MaybeIncorrect); - } else { - let msg = - format!("Unicode character '{}' ({}) looks like '{}' ({}), but it is not", - ch, u_name, ascii_char, ascii_name); - err.span_suggestion( - span, - &msg, - ascii_char.to_string(), - Applicability::MaybeIncorrect); - } - true - }, - None => { - let msg = format!("substitution character not found for '{}'", ch); - reader.sess.span_diagnostic.span_bug_no_panic(span, &msg); - false - } + ('＞', "Fullwidth Greater-Than Sign", '>'), +]; + +// FIXME: the lexer could be used to turn the ASCII version of unicode homoglyphs, instead of +// keeping the substitution token in this table. Ideally, this should be inside `rustc_lexer`. +// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add +// fancier error recovery to it, as there will be less overall work to do this way. +const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[ + (' ', "Space", Some(token::Whitespace)), + ('_', "Underscore", Some(token::Ident(kw::Underscore, false))), + ('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))), + (',', "Comma", Some(token::Comma)), + (';', "Semicolon", Some(token::Semi)), + (':', "Colon", Some(token::Colon)), + ('!', "Exclamation Mark", Some(token::Not)), + ('?', "Question Mark", Some(token::Question)), + ('.', "Period", Some(token::Dot)), + ('(', "Left Parenthesis", Some(token::OpenDelim(token::Paren))), + (')', "Right Parenthesis", Some(token::CloseDelim(token::Paren))), + ('[', "Left Square Bracket", Some(token::OpenDelim(token::Bracket))), + (']', "Right Square Bracket", Some(token::CloseDelim(token::Bracket))), + ('{', "Left Curly Brace", Some(token::OpenDelim(token::Brace))), + ('}', "Right Curly Brace", Some(token::CloseDelim(token::Brace))), + ('*', "Asterisk", Some(token::BinOp(token::Star))), + ('/', "Slash", Some(token::BinOp(token::Slash))), + ('\\', "Backslash", None), + ('&', "Ampersand", Some(token::BinOp(token::And))), + ('+', "Plus Sign", Some(token::BinOp(token::Plus))), + ('<', "Less-Than Sign", Some(token::Lt)), + ('=', "Equals Sign", Some(token::Eq)), + ('>', "Greater-Than Sign", Some(token::Gt)), + // FIXME: Literals are already lexed by this point, so we can't recover gracefully just by + // spitting the correct token out. + ('\'', "Single Quote", None), + ('"', "Quotation Mark", None), +]; + +crate fn check_for_substitution<'a>( + reader: &StringReader<'a>, + pos: BytePos, + ch: char, + err: &mut DiagnosticBuilder<'a>, +) -> Option<token::TokenKind> { + let (u_name, ascii_char) = match UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch) { + Some(&(_u_char, u_name, ascii_char)) => (u_name, ascii_char), + None => return None, + }; + + let span = Span::new(pos, pos + Pos::from_usize(ch.len_utf8()), NO_EXPANSION); + + let (ascii_name, token) = match ASCII_ARRAY.iter().find(|&&(c, _, _)| c == ascii_char) { + Some((_ascii_char, ascii_name, token)) => (ascii_name, token), + None => { + let msg = format!("substitution character not found for '{}'", ch); + reader.sess.span_diagnostic.span_bug_no_panic(span, &msg); + return None; } - }).unwrap_or(false) + }; + + // special help suggestion for "directed" double quotes + if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') { + let msg = format!( + "Unicode characters '“' (Left Double Quotation Mark) and \ + '”' (Right Double Quotation Mark) look like '{}' ({}), but are not", + ascii_char, ascii_name + ); + err.span_suggestion( + Span::new( + pos, + pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()), + NO_EXPANSION, + ), + &msg, + format!("\"{}\"", s), + Applicability::MaybeIncorrect, + ); + } else { + let msg = format!( + "Unicode character '{}' ({}) looks like '{}' ({}), but it is not", + ch, u_name, ascii_char, ascii_name + ); + err.span_suggestion(span, &msg, ascii_char.to_string(), Applicability::MaybeIncorrect); + } + token.clone() +} + +/// Extract string if found at current position with given delimiters +fn peek_delimited(text: &str, from_ch: char, to_ch: char) -> Option<&str> { + let mut chars = text.chars(); + let first_char = chars.next()?; + if first_char != from_ch { + return None; + } + let last_char_idx = chars.as_str().find(to_ch)?; + Some(&chars.as_str()[..last_char_idx]) } diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs new file mode 100644 index 00000000000..6409acba573 --- /dev/null +++ b/src/libsyntax/parse/literal.rs @@ -0,0 +1,470 @@ +//! Code related to parsing literals. + +use crate::ast::{self, Lit, LitKind}; +use crate::parse::parser::Parser; +use crate::parse::PResult; +use crate::parse::token::{self, Token, TokenKind}; +use crate::print::pprust; +use crate::symbol::{kw, sym, Symbol}; +use crate::tokenstream::{TokenStream, TokenTree}; + +use errors::{Applicability, Handler}; +use log::debug; +use rustc_data_structures::sync::Lrc; +use syntax_pos::Span; +use rustc_lexer::unescape::{unescape_char, unescape_byte}; +use rustc_lexer::unescape::{unescape_str, unescape_byte_str}; +use rustc_lexer::unescape::{unescape_raw_str, unescape_raw_byte_str}; + +use std::ascii; + +crate enum LitError { + NotLiteral, + LexerError, + InvalidSuffix, + InvalidIntSuffix, + InvalidFloatSuffix, + NonDecimalFloat(u32), + IntTooLarge, +} + +impl LitError { + fn report(&self, diag: &Handler, lit: token::Lit, span: Span) { + let token::Lit { kind, suffix, .. } = lit; + match *self { + // `NotLiteral` is not an error by itself, so we don't report + // it and give the parser opportunity to try something else. + LitError::NotLiteral => {} + // `LexerError` *is* an error, but it was already reported + // by lexer, so here we don't report it the second time. + LitError::LexerError => {} + LitError::InvalidSuffix => { + expect_no_suffix( + diag, span, &format!("{} {} literal", kind.article(), kind.descr()), suffix + ); + } + LitError::InvalidIntSuffix => { + let suf = suffix.expect("suffix error with no suffix").as_str(); + if looks_like_width_suffix(&['i', 'u'], &suf) { + // If it looks like a width, try to be helpful. + let msg = format!("invalid width `{}` for integer literal", &suf[1..]); + diag.struct_span_err(span, &msg) + .help("valid widths are 8, 16, 32, 64 and 128") + .emit(); + } else { + let msg = format!("invalid suffix `{}` for integer literal", suf); + diag.struct_span_err(span, &msg) + .span_label(span, format!("invalid suffix `{}`", suf)) + .help("the suffix must be one of the integral types (`u32`, `isize`, etc)") + .emit(); + } + } + LitError::InvalidFloatSuffix => { + let suf = suffix.expect("suffix error with no suffix").as_str(); + if looks_like_width_suffix(&['f'], &suf) { + // If it looks like a width, try to be helpful. + let msg = format!("invalid width `{}` for float literal", &suf[1..]); + diag.struct_span_err(span, &msg) + .help("valid widths are 32 and 64") + .emit(); + } else { + let msg = format!("invalid suffix `{}` for float literal", suf); + diag.struct_span_err(span, &msg) + .span_label(span, format!("invalid suffix `{}`", suf)) + .help("valid suffixes are `f32` and `f64`") + .emit(); + } + } + LitError::NonDecimalFloat(base) => { + let descr = match base { + 16 => "hexadecimal", + 8 => "octal", + 2 => "binary", + _ => unreachable!(), + }; + diag.struct_span_err(span, &format!("{} float literal is not supported", descr)) + .span_label(span, "not supported") + .emit(); + } + LitError::IntTooLarge => { + diag.struct_span_err(span, "integer literal is too large") + .emit(); + } + } + } +} + +impl LitKind { + /// Converts literal token into a semantic literal. + fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> { + let token::Lit { kind, symbol, suffix } = lit; + if suffix.is_some() && !kind.may_have_suffix() { + return Err(LitError::InvalidSuffix); + } + + Ok(match kind { + token::Bool => { + assert!(symbol == kw::True || symbol == kw::False); + LitKind::Bool(symbol == kw::True) + } + token::Byte => return unescape_byte(&symbol.as_str()) + .map(LitKind::Byte).map_err(|_| LitError::LexerError), + token::Char => return unescape_char(&symbol.as_str()) + .map(LitKind::Char).map_err(|_| LitError::LexerError), + + // There are some valid suffixes for integer and float literals, + // so all the handling is done internally. + token::Integer => return integer_lit(symbol, suffix), + token::Float => return float_lit(symbol, suffix), + + token::Str => { + // If there are no characters requiring special treatment we can + // reuse the symbol from the token. Otherwise, we must generate a + // new symbol because the string in the LitKind is different to the + // string in the token. + let s = symbol.as_str(); + let symbol = if s.contains(&['\\', '\r'][..]) { + let mut buf = String::with_capacity(s.len()); + let mut error = Ok(()); + unescape_str(&s, &mut |_, unescaped_char| { + match unescaped_char { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + Symbol::intern(&buf) + } else { + symbol + }; + LitKind::Str(symbol, ast::StrStyle::Cooked) + } + token::StrRaw(n) => { + // Ditto. + let s = symbol.as_str(); + let symbol = if s.contains('\r') { + let mut buf = String::with_capacity(s.len()); + let mut error = Ok(()); + unescape_raw_str(&s, &mut |_, unescaped_char| { + match unescaped_char { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + Symbol::intern(&buf) + } else { + symbol + }; + LitKind::Str(symbol, ast::StrStyle::Raw(n)) + } + token::ByteStr => { + let s = symbol.as_str(); + let mut buf = Vec::with_capacity(s.len()); + let mut error = Ok(()); + unescape_byte_str(&s, &mut |_, unescaped_byte| { + match unescaped_byte { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + LitKind::ByteStr(Lrc::new(buf)) + } + token::ByteStrRaw(_) => { + let s = symbol.as_str(); + let bytes = if s.contains('\r') { + let mut buf = Vec::with_capacity(s.len()); + let mut error = Ok(()); + unescape_raw_byte_str(&s, &mut |_, unescaped_byte| { + match unescaped_byte { + Ok(c) => buf.push(c), + Err(_) => error = Err(LitError::LexerError), + } + }); + error?; + buf.shrink_to_fit(); + buf + } else { + symbol.to_string().into_bytes() + }; + + LitKind::ByteStr(Lrc::new(bytes)) + }, + token::Err => LitKind::Err(symbol), + }) + } + + /// Attempts to recover a token from semantic literal. + /// This function is used when the original token doesn't exist (e.g. the literal is created + /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). + pub fn to_lit_token(&self) -> token::Lit { + let (kind, symbol, suffix) = match *self { + LitKind::Str(symbol, ast::StrStyle::Cooked) => { + // Don't re-intern unless the escaped string is different. + let s = &symbol.as_str(); + let escaped = s.escape_default().to_string(); + let symbol = if escaped == *s { symbol } else { Symbol::intern(&escaped) }; + (token::Str, symbol, None) + } + LitKind::Str(symbol, ast::StrStyle::Raw(n)) => { + (token::StrRaw(n), symbol, None) + } + LitKind::ByteStr(ref bytes) => { + let string = bytes.iter().cloned().flat_map(ascii::escape_default) + .map(Into::<char>::into).collect::<String>(); + (token::ByteStr, Symbol::intern(&string), None) + } + LitKind::Byte(byte) => { + let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect(); + (token::Byte, Symbol::intern(&string), None) + } + LitKind::Char(ch) => { + let string: String = ch.escape_default().map(Into::<char>::into).collect(); + (token::Char, Symbol::intern(&string), None) + } + LitKind::Int(n, ty) => { + let suffix = match ty { + ast::LitIntType::Unsigned(ty) => Some(ty.to_symbol()), + ast::LitIntType::Signed(ty) => Some(ty.to_symbol()), + ast::LitIntType::Unsuffixed => None, + }; + (token::Integer, sym::integer(n), suffix) + } + LitKind::Float(symbol, ty) => { + (token::Float, symbol, Some(ty.to_symbol())) + } + LitKind::FloatUnsuffixed(symbol) => { + (token::Float, symbol, None) + } + LitKind::Bool(value) => { + let symbol = if value { kw::True } else { kw::False }; + (token::Bool, symbol, None) + } + LitKind::Err(symbol) => { + (token::Err, symbol, None) + } + }; + + token::Lit::new(kind, symbol, suffix) + } +} + +impl Lit { + /// Converts literal token into an AST literal. + fn from_lit_token(token: token::Lit, span: Span) -> Result<Lit, LitError> { + Ok(Lit { token, node: LitKind::from_lit_token(token)?, span }) + } + + /// Converts arbitrary token into an AST literal. + crate fn from_token(token: &Token) -> Result<Lit, LitError> { + let lit = match token.kind { + token::Ident(name, false) if name == kw::True || name == kw::False => + token::Lit::new(token::Bool, name, None), + token::Literal(lit) => + lit, + token::Interpolated(ref nt) => { + if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt { + if let ast::ExprKind::Lit(lit) = &expr.node { + return Ok(lit.clone()); + } + } + return Err(LitError::NotLiteral); + } + _ => return Err(LitError::NotLiteral) + }; + + Lit::from_lit_token(lit, token.span) + } + + /// Attempts to recover an AST literal from semantic literal. + /// This function is used when the original token doesn't exist (e.g. the literal is created + /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). + pub fn from_lit_kind(node: LitKind, span: Span) -> Lit { + Lit { token: node.to_lit_token(), node, span } + } + + /// Losslessly convert an AST literal into a token stream. + crate fn tokens(&self) -> TokenStream { + let token = match self.token.kind { + token::Bool => token::Ident(self.token.symbol, false), + _ => token::Literal(self.token), + }; + TokenTree::token(token, self.span).into() + } +} + +impl<'a> Parser<'a> { + /// Matches `lit = true | false | token_lit`. + crate fn parse_lit(&mut self) -> PResult<'a, Lit> { + let mut recovered = None; + if self.token == token::Dot { + // Attempt to recover `.4` as `0.4`. + recovered = self.look_ahead(1, |next_token| { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) + = next_token.kind { + if self.token.span.hi() == next_token.span.lo() { + let s = String::from("0.") + &symbol.as_str(); + let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix); + return Some(Token::new(kind, self.token.span.to(next_token.span))); + } + } + None + }); + if let Some(token) = &recovered { + self.bump(); + self.diagnostic() + .struct_span_err(token.span, "float literals must have an integer part") + .span_suggestion( + token.span, + "must have an integer part", + pprust::token_to_string(token), + Applicability::MachineApplicable, + ) + .emit(); + } + } + + let token = recovered.as_ref().unwrap_or(&self.token); + match Lit::from_token(token) { + Ok(lit) => { + self.bump(); + Ok(lit) + } + Err(LitError::NotLiteral) => { + let msg = format!("unexpected token: {}", self.this_token_descr()); + Err(self.span_fatal(token.span, &msg)) + } + Err(err) => { + let (lit, span) = (token.expect_lit(), token.span); + self.bump(); + err.report(&self.sess.span_diagnostic, lit, span); + // Pack possible quotes and prefixes from the original literal into + // the error literal's symbol so they can be pretty-printed faithfully. + let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None); + let symbol = Symbol::intern(&suffixless_lit.to_string()); + let lit = token::Lit::new(token::Err, symbol, lit.suffix); + Lit::from_lit_token(lit, span).map_err(|_| unreachable!()) + } + } + } +} + +crate fn expect_no_suffix(diag: &Handler, sp: Span, kind: &str, suffix: Option<Symbol>) { + if let Some(suf) = suffix { + let mut err = if kind == "a tuple index" && + [sym::i32, sym::u32, sym::isize, sym::usize].contains(&suf) { + // #59553: warn instead of reject out of hand to allow the fix to percolate + // through the ecosystem when people fix their macros + let mut err = diag.struct_span_warn( + sp, + &format!("suffixes on {} are invalid", kind), + ); + err.note(&format!( + "`{}` is *temporarily* accepted on tuple index fields as it was \ + incorrectly accepted on stable for a few releases", + suf, + )); + err.help( + "on proc macros, you'll want to use `syn::Index::from` or \ + `proc_macro::Literal::*_unsuffixed` for code that will desugar \ + to tuple field access", + ); + err.note( + "for more context, see https://github.com/rust-lang/rust/issues/60210", + ); + err + } else { + diag.struct_span_err(sp, &format!("suffixes on {} are invalid", kind)) + }; + err.span_label(sp, format!("invalid suffix `{}`", suf)); + err.emit(); + } +} + +// Checks if `s` looks like i32 or u1234 etc. +fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { + s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit()) +} + +fn strip_underscores(symbol: Symbol) -> Symbol { + // Do not allocate a new string unless necessary. + let s = symbol.as_str(); + if s.contains('_') { + let mut s = s.to_string(); + s.retain(|c| c != '_'); + return Symbol::intern(&s); + } + symbol +} + +fn filtered_float_lit(symbol: Symbol, suffix: Option<Symbol>, base: u32) + -> Result<LitKind, LitError> { + debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base); + if base != 10 { + return Err(LitError::NonDecimalFloat(base)); + } + Ok(match suffix { + Some(suf) => match suf { + sym::f32 => LitKind::Float(symbol, ast::FloatTy::F32), + sym::f64 => LitKind::Float(symbol, ast::FloatTy::F64), + _ => return Err(LitError::InvalidFloatSuffix), + } + None => LitKind::FloatUnsuffixed(symbol) + }) +} + +fn float_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> { + debug!("float_lit: {:?}, {:?}", symbol, suffix); + filtered_float_lit(strip_underscores(symbol), suffix, 10) +} + +fn integer_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> { + debug!("integer_lit: {:?}, {:?}", symbol, suffix); + let symbol = strip_underscores(symbol); + let s = symbol.as_str(); + + let mut base = 10; + if s.len() > 1 && s.as_bytes()[0] == b'0' { + match s.as_bytes()[1] { + b'x' => base = 16, + b'o' => base = 8, + b'b' => base = 2, + _ => {} + } + } + + let ty = match suffix { + Some(suf) => match suf { + sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize), + sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8), + sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16), + sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32), + sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64), + sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128), + sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize), + sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8), + sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16), + sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32), + sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64), + sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128), + // `1f64` and `2f32` etc. are valid float literals, and + // `fxxx` looks more like an invalid float literal than invalid integer literal. + _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base), + _ => return Err(LitError::InvalidIntSuffix), + } + _ => ast::LitIntType::Unsuffixed + }; + + let s = &s[if base != 10 { 2 } else { 0 } ..]; + u128::from_str_radix(s, base).map(|i| LitKind::Int(i, ty)).map_err(|_| { + // Small bases are lexed as if they were base 10, e.g, the string + // might be `0b10201`. This will cause the conversion above to fail, + // but these kinds of errors are already reported by the lexer. + let from_lexer = + base < 10 && s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base)); + if from_lexer { LitError::LexerError } else { LitError::IntTooLarge } + }) +} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 371e8fe5cf6..80aa7a35266 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -5,39 +5,45 @@ use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; use crate::source_map::{SourceMap, FilePathMapping}; use crate::feature_gate::UnstableFeatures; use crate::parse::parser::Parser; -use crate::symbol::Symbol; -use crate::syntax::parse::parser::emit_unclosed_delims; +use crate::parse::parser::emit_unclosed_delims; +use crate::parse::token::TokenKind; use crate::tokenstream::{TokenStream, TokenTree}; use crate::diagnostics::plugin::ErrorMap; -use crate::print::pprust::token_to_string; +use crate::print::pprust; +use crate::symbol::Symbol; -use errors::{FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder}; -use rustc_data_structures::sync::{Lrc, Lock}; +use errors::{Applicability, FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder}; +use rustc_data_structures::sync::{Lrc, Lock, Once}; use syntax_pos::{Span, SourceFile, FileName, MultiSpan}; -use log::debug; +use syntax_pos::edition::Edition; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashSet, FxHashMap}; use std::borrow::Cow; -use std::iter; use std::path::{Path, PathBuf}; use std::str; -pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; +#[cfg(test)] +mod tests; #[macro_use] pub mod parser; - +pub mod attr; pub mod lexer; pub mod token; -pub mod attr; -pub mod classify; +crate mod classify; +crate mod diagnostics; +crate mod literal; +crate mod unescape_error_reporting; + +pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; /// Info about a parsing session. pub struct ParseSess { pub span_diagnostic: Handler, pub unstable_features: UnstableFeatures, pub config: CrateConfig, + pub edition: Edition, pub missing_fragment_specifiers: Lock<FxHashSet<Span>>, /// Places where raw identifiers were used. This is used for feature-gating raw identifiers. pub raw_identifier_spans: Lock<Vec<Span>>, @@ -47,6 +53,16 @@ pub struct ParseSess { included_mod_stack: Lock<Vec<PathBuf>>, source_map: Lrc<SourceMap>, pub buffered_lints: Lock<Vec<BufferedEarlyLint>>, + /// Contains the spans of block expressions that could have been incomplete based on the + /// operation token that followed it, but that the parser cannot identify without further + /// analysis. + pub ambiguous_block_expr_parse: Lock<FxHashMap<Span, Span>>, + pub param_attr_spans: Lock<Vec<Span>>, + // Places where `let` exprs were used and should be feature gated according to `let_chains`. + pub let_chains_spans: Lock<Vec<Span>>, + // Places where `async || ..` exprs were used and should be feature gated. + pub async_closure_spans: Lock<Vec<Span>>, + pub injected_crate_name: Once<Symbol>, } impl ParseSess { @@ -70,6 +86,12 @@ impl ParseSess { included_mod_stack: Lock::new(vec![]), source_map, buffered_lints: Lock::new(vec![]), + edition: Edition::from_session(), + ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), + param_attr_spans: Lock::new(Vec::new()), + let_chains_spans: Lock::new(Vec::new()), + async_closure_spans: Lock::new(Vec::new()), + injected_crate_name: Once::new(), } } @@ -93,6 +115,24 @@ impl ParseSess { }); }); } + + /// Extend an error with a suggestion to wrap an expression with parentheses to allow the + /// parser to continue parsing the following operation as part of the same expression. + pub fn expr_parentheses_needed( + &self, + err: &mut DiagnosticBuilder<'_>, + span: Span, + alt_snippet: Option<String>, + ) { + if let Some(snippet) = self.source_map().span_to_snippet(span).ok().or(alt_snippet) { + err.span_suggestion( + span, + "parentheses are required to parse this as an expression", + format!("({})", snippet), + Applicability::MachineApplicable, + ); + } + } } #[derive(Clone)] @@ -210,10 +250,10 @@ fn maybe_source_file_to_parser( ) -> Result<Parser<'_>, Vec<Diagnostic>> { let end_pos = source_file.end_pos; let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?; - let mut parser = stream_to_parser(sess, stream); + let mut parser = stream_to_parser(sess, stream, None); parser.unclosed_delims = unclosed_delims; - if parser.token == token::Eof && parser.span.is_dummy() { - parser.span = Span::new(end_pos, end_pos, parser.span.ctxt()); + if parser.token == token::Eof && parser.token.span.is_dummy() { + parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt()); } Ok(parser) @@ -222,7 +262,7 @@ fn maybe_source_file_to_parser( // must preserve old name for now, because quote! from the *existing* // compiler expands into it pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser<'_> { - stream_to_parser(sess, tts.into_iter().collect()) + stream_to_parser(sess, tts.into_iter().collect(), crate::MACRO_ARGUMENTS) } @@ -267,25 +307,25 @@ pub fn source_file_to_stream( } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from -/// parsing the token tream. +/// parsing the token stream. pub fn maybe_file_to_stream( sess: &ParseSess, source_file: Lrc<SourceFile>, override_span: Option<Span>, ) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> { - let mut srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?; - srdr.real_token(); + let srdr = lexer::StringReader::new(sess, source_file, override_span); + let (token_trees, unmatched_braces) = srdr.into_token_trees(); - match srdr.parse_all_token_trees() { - Ok(stream) => Ok((stream, srdr.unmatched_braces)), + match token_trees { + Ok(stream) => Ok((stream, unmatched_braces)), Err(err) => { let mut buffer = Vec::with_capacity(1); err.buffer(&mut buffer); // Not using `emit_unclosed_delims` to use `db.buffer` - for unmatched in srdr.unmatched_braces { + for unmatched in unmatched_braces { let mut db = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!( "incorrect close delimiter: `{}`", - token_to_string(&token::Token::CloseDelim(unmatched.found_delim)), + pprust::token_kind_to_string(&token::CloseDelim(unmatched.found_delim)), )); db.span_label(unmatched.found_span, "incorrect close delimiter"); if let Some(sp) = unmatched.candidate_span { @@ -302,477 +342,43 @@ pub fn maybe_file_to_stream( } /// Given stream and the `ParseSess`, produces a parser. -pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> { - Parser::new(sess, stream, None, true, false) -} - -/// Parses a string representing a character literal into its final form. -/// Rather than just accepting/rejecting a given literal, unescapes it as -/// well. Can take any slice prefixed by a character escape. Returns the -/// character and the number of characters consumed. -fn char_lit(lit: &str, diag: Option<(Span, &Handler)>) -> (char, isize) { - use std::char; - - // Handle non-escaped chars first. - if lit.as_bytes()[0] != b'\\' { - // If the first byte isn't '\\' it might part of a multi-byte char, so - // get the char with chars(). - let c = lit.chars().next().unwrap(); - return (c, 1); - } - - // Handle escaped chars. - match lit.as_bytes()[1] as char { - '"' => ('"', 2), - 'n' => ('\n', 2), - 'r' => ('\r', 2), - 't' => ('\t', 2), - '\\' => ('\\', 2), - '\'' => ('\'', 2), - '0' => ('\0', 2), - 'x' => { - let v = u32::from_str_radix(&lit[2..4], 16).unwrap(); - let c = char::from_u32(v).unwrap(); - (c, 4) - } - 'u' => { - assert_eq!(lit.as_bytes()[2], b'{'); - let idx = lit.find('}').unwrap(); - - // All digits and '_' are ascii, so treat each byte as a char. - let mut v: u32 = 0; - for c in lit[3..idx].bytes() { - let c = char::from(c); - if c != '_' { - let x = c.to_digit(16).unwrap(); - v = v.checked_mul(16).unwrap().checked_add(x).unwrap(); - } - } - let c = char::from_u32(v).unwrap_or_else(|| { - if let Some((span, diag)) = diag { - let mut diag = diag.struct_span_err(span, "invalid unicode character escape"); - if v > 0x10FFFF { - diag.help("unicode escape must be at most 10FFFF").emit(); - } else { - diag.help("unicode escape must not be a surrogate").emit(); - } - } - '\u{FFFD}' - }); - (c, (idx + 1) as isize) - } - _ => panic!("lexer should have rejected a bad character escape {}", lit) - } -} - -/// Parses a string representing a string literal into its final form. Does unescaping. -pub fn str_lit(lit: &str, diag: Option<(Span, &Handler)>) -> String { - debug!("str_lit: given {}", lit.escape_default()); - let mut res = String::with_capacity(lit.len()); - - let error = |i| format!("lexer should have rejected {} at {}", lit, i); - - /// Eat everything up to a non-whitespace. - fn eat<'a>(it: &mut iter::Peekable<str::CharIndices<'a>>) { - loop { - match it.peek().map(|x| x.1) { - Some(' ') | Some('\n') | Some('\r') | Some('\t') => { - it.next(); - }, - _ => { break; } - } - } - } - - let mut chars = lit.char_indices().peekable(); - while let Some((i, c)) = chars.next() { - match c { - '\\' => { - let ch = chars.peek().unwrap_or_else(|| { - panic!("{}", error(i)) - }).1; - - if ch == '\n' { - eat(&mut chars); - } else if ch == '\r' { - chars.next(); - let ch = chars.peek().unwrap_or_else(|| { - panic!("{}", error(i)) - }).1; - - if ch != '\n' { - panic!("lexer accepted bare CR"); - } - eat(&mut chars); - } else { - // otherwise, a normal escape - let (c, n) = char_lit(&lit[i..], diag); - for _ in 0..n - 1 { // we don't need to move past the first \ - chars.next(); - } - res.push(c); - } - }, - '\r' => { - let ch = chars.peek().unwrap_or_else(|| { - panic!("{}", error(i)) - }).1; - - if ch != '\n' { - panic!("lexer accepted bare CR"); - } - chars.next(); - res.push('\n'); - } - c => res.push(c), - } - } - - res.shrink_to_fit(); // probably not going to do anything, unless there was an escape. - debug!("parse_str_lit: returning {}", res); - res -} - -/// Parses a string representing a raw string literal into its final form. The -/// only operation this does is convert embedded CRLF into a single LF. -fn raw_str_lit(lit: &str) -> String { - debug!("raw_str_lit: given {}", lit.escape_default()); - let mut res = String::with_capacity(lit.len()); - - let mut chars = lit.chars().peekable(); - while let Some(c) = chars.next() { - if c == '\r' { - if *chars.peek().unwrap() != '\n' { - panic!("lexer accepted bare CR"); - } - chars.next(); - res.push('\n'); - } else { - res.push(c); - } - } - - res.shrink_to_fit(); - res -} - -// check if `s` looks like i32 or u1234 etc. -fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool { - s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit()) -} - -macro_rules! err { - ($opt_diag:expr, |$span:ident, $diag:ident| $($body:tt)*) => { - match $opt_diag { - Some(($span, $diag)) => { $($body)* } - None => return None, - } - } -} - -crate fn lit_token(lit: token::Lit, suf: Option<Symbol>, diag: Option<(Span, &Handler)>) - -> (bool /* suffix illegal? */, Option<ast::LitKind>) { - use ast::LitKind; - - match lit { - token::Byte(i) => (true, Some(LitKind::Byte(byte_lit(&i.as_str()).0))), - token::Char(i) => (true, Some(LitKind::Char(char_lit(&i.as_str(), diag).0))), - token::Err(i) => (true, Some(LitKind::Err(i))), - - // There are some valid suffixes for integer and float literals, - // so all the handling is done internally. - token::Integer(s) => (false, integer_lit(&s.as_str(), suf, diag)), - token::Float(s) => (false, float_lit(&s.as_str(), suf, diag)), - - token::Str_(mut sym) => { - // If there are no characters requiring special treatment we can - // reuse the symbol from the Token. Otherwise, we must generate a - // new symbol because the string in the LitKind is different to the - // string in the Token. - let s = &sym.as_str(); - if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') { - sym = Symbol::intern(&str_lit(s, diag)); - } - (true, Some(LitKind::Str(sym, ast::StrStyle::Cooked))) - } - token::StrRaw(mut sym, n) => { - // Ditto. - let s = &sym.as_str(); - if s.contains('\r') { - sym = Symbol::intern(&raw_str_lit(s)); - } - (true, Some(LitKind::Str(sym, ast::StrStyle::Raw(n)))) - } - token::ByteStr(i) => { - (true, Some(LitKind::ByteStr(byte_str_lit(&i.as_str())))) - } - token::ByteStrRaw(i, _) => { - (true, Some(LitKind::ByteStr(Lrc::new(i.to_string().into_bytes())))) - } - } -} - -fn filtered_float_lit(data: Symbol, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>) - -> Option<ast::LitKind> { - debug!("filtered_float_lit: {}, {:?}", data, suffix); - let suffix = match suffix { - Some(suffix) => suffix, - None => return Some(ast::LitKind::FloatUnsuffixed(data)), - }; - - Some(match &*suffix.as_str() { - "f32" => ast::LitKind::Float(data, ast::FloatTy::F32), - "f64" => ast::LitKind::Float(data, ast::FloatTy::F64), - suf => { - err!(diag, |span, diag| { - if suf.len() >= 2 && looks_like_width_suffix(&['f'], suf) { - // if it looks like a width, lets try to be helpful. - let msg = format!("invalid width `{}` for float literal", &suf[1..]); - diag.struct_span_err(span, &msg).help("valid widths are 32 and 64").emit() - } else { - let msg = format!("invalid suffix `{}` for float literal", suf); - diag.struct_span_err(span, &msg) - .span_label(span, format!("invalid suffix `{}`", suf)) - .help("valid suffixes are `f32` and `f64`") - .emit(); - } - }); - - ast::LitKind::FloatUnsuffixed(data) - } - }) -} -fn float_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>) - -> Option<ast::LitKind> { - debug!("float_lit: {:?}, {:?}", s, suffix); - // FIXME #2252: bounds checking float literals is deferred until trans - - // Strip underscores without allocating a new String unless necessary. - let s2; - let s = if s.chars().any(|c| c == '_') { - s2 = s.chars().filter(|&c| c != '_').collect::<String>(); - &s2 - } else { - s - }; - - filtered_float_lit(Symbol::intern(s), suffix, diag) -} - -/// Parses a string representing a byte literal into its final form. Similar to `char_lit`. -fn byte_lit(lit: &str) -> (u8, usize) { - let err = |i| format!("lexer accepted invalid byte literal {} step {}", lit, i); - - if lit.len() == 1 { - (lit.as_bytes()[0], 1) - } else { - assert_eq!(lit.as_bytes()[0], b'\\', "{}", err(0)); - let b = match lit.as_bytes()[1] { - b'"' => b'"', - b'n' => b'\n', - b'r' => b'\r', - b't' => b'\t', - b'\\' => b'\\', - b'\'' => b'\'', - b'0' => b'\0', - _ => { - match u64::from_str_radix(&lit[2..4], 16).ok() { - Some(c) => - if c > 0xFF { - panic!(err(2)) - } else { - return (c as u8, 4) - }, - None => panic!(err(3)) - } - } - }; - (b, 2) - } -} - -fn byte_str_lit(lit: &str) -> Lrc<Vec<u8>> { - let mut res = Vec::with_capacity(lit.len()); - - let error = |i| panic!("lexer should have rejected {} at {}", lit, i); - - /// Eat everything up to a non-whitespace. - fn eat<I: Iterator<Item=(usize, u8)>>(it: &mut iter::Peekable<I>) { - loop { - match it.peek().map(|x| x.1) { - Some(b' ') | Some(b'\n') | Some(b'\r') | Some(b'\t') => { - it.next(); - }, - _ => { break; } - } - } - } - - // byte string literals *must* be ASCII, but the escapes don't have to be - let mut chars = lit.bytes().enumerate().peekable(); - loop { - match chars.next() { - Some((i, b'\\')) => { - match chars.peek().unwrap_or_else(|| error(i)).1 { - b'\n' => eat(&mut chars), - b'\r' => { - chars.next(); - if chars.peek().unwrap_or_else(|| error(i)).1 != b'\n' { - panic!("lexer accepted bare CR"); - } - eat(&mut chars); - } - _ => { - // otherwise, a normal escape - let (c, n) = byte_lit(&lit[i..]); - // we don't need to move past the first \ - for _ in 0..n - 1 { - chars.next(); - } - res.push(c); - } - } - }, - Some((i, b'\r')) => { - if chars.peek().unwrap_or_else(|| error(i)).1 != b'\n' { - panic!("lexer accepted bare CR"); - } - chars.next(); - res.push(b'\n'); - } - Some((_, c)) => res.push(c), - None => break, - } - } - - Lrc::new(res) -} - -fn integer_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>) - -> Option<ast::LitKind> { - // s can only be ascii, byte indexing is fine - - // Strip underscores without allocating a new String unless necessary. - let s2; - let mut s = if s.chars().any(|c| c == '_') { - s2 = s.chars().filter(|&c| c != '_').collect::<String>(); - &s2 - } else { - s - }; - - debug!("integer_lit: {}, {:?}", s, suffix); - - let mut base = 10; - let orig = s; - let mut ty = ast::LitIntType::Unsuffixed; - - if s.starts_with('0') && s.len() > 1 { - match s.as_bytes()[1] { - b'x' => base = 16, - b'o' => base = 8, - b'b' => base = 2, - _ => { } - } - } - - // 1f64 and 2f32 etc. are valid float literals. - if let Some(suf) = suffix { - if looks_like_width_suffix(&['f'], &suf.as_str()) { - let err = match base { - 16 => Some("hexadecimal float literal is not supported"), - 8 => Some("octal float literal is not supported"), - 2 => Some("binary float literal is not supported"), - _ => None, - }; - if let Some(err) = err { - err!(diag, |span, diag| { - diag.struct_span_err(span, err) - .span_label(span, "not supported") - .emit(); - }); - } - return filtered_float_lit(Symbol::intern(s), Some(suf), diag) - } - } - - if base != 10 { - s = &s[2..]; - } - - if let Some(suf) = suffix { - if suf.as_str().is_empty() { - err!(diag, |span, diag| diag.span_bug(span, "found empty literal suffix in Some")); - } - ty = match &*suf.as_str() { - "isize" => ast::LitIntType::Signed(ast::IntTy::Isize), - "i8" => ast::LitIntType::Signed(ast::IntTy::I8), - "i16" => ast::LitIntType::Signed(ast::IntTy::I16), - "i32" => ast::LitIntType::Signed(ast::IntTy::I32), - "i64" => ast::LitIntType::Signed(ast::IntTy::I64), - "i128" => ast::LitIntType::Signed(ast::IntTy::I128), - "usize" => ast::LitIntType::Unsigned(ast::UintTy::Usize), - "u8" => ast::LitIntType::Unsigned(ast::UintTy::U8), - "u16" => ast::LitIntType::Unsigned(ast::UintTy::U16), - "u32" => ast::LitIntType::Unsigned(ast::UintTy::U32), - "u64" => ast::LitIntType::Unsigned(ast::UintTy::U64), - "u128" => ast::LitIntType::Unsigned(ast::UintTy::U128), - suf => { - // i<digits> and u<digits> look like widths, so lets - // give an error message along those lines - err!(diag, |span, diag| { - if looks_like_width_suffix(&['i', 'u'], suf) { - let msg = format!("invalid width `{}` for integer literal", &suf[1..]); - diag.struct_span_err(span, &msg) - .help("valid widths are 8, 16, 32, 64 and 128") - .emit(); - } else { - let msg = format!("invalid suffix `{}` for numeric literal", suf); - diag.struct_span_err(span, &msg) - .span_label(span, format!("invalid suffix `{}`", suf)) - .help("the suffix must be one of the integral types \ - (`u32`, `isize`, etc)") - .emit(); - } - }); - - ty - } - } - } - - debug!("integer_lit: the type is {:?}, base {:?}, the new string is {:?}, the original \ - string was {:?}, the original suffix was {:?}", ty, base, s, orig, suffix); - - Some(match u128::from_str_radix(s, base) { - Ok(r) => ast::LitKind::Int(r, ty), - Err(_) => { - // small bases are lexed as if they were base 10, e.g, the string - // might be `0b10201`. This will cause the conversion above to fail, - // but these cases have errors in the lexer: we don't want to emit - // two errors, and we especially don't want to emit this error since - // it isn't necessarily true. - let already_errored = base < 10 && - s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base)); - - if !already_errored { - err!(diag, |span, diag| diag.span_err(span, "int literal is too large")); - } - ast::LitKind::Int(0, ty) - } - }) +pub fn stream_to_parser<'a>( + sess: &'a ParseSess, + stream: TokenStream, + subparser_name: Option<&'static str>, +) -> Parser<'a> { + Parser::new(sess, stream, None, true, false, subparser_name) +} + +/// Given stream, the `ParseSess` and the base directory, produces a parser. +/// +/// Use this function when you are creating a parser from the token stream +/// and also care about the current working directory of the parser (e.g., +/// you are trying to resolve modules defined inside a macro invocation). +/// +/// # Note +/// +/// The main usage of this function is outside of rustc, for those who uses +/// libsyntax as a library. Please do not remove this function while refactoring +/// just because it is not used in rustc codebase! +pub fn stream_to_parser_with_base_dir<'a>( + sess: &'a ParseSess, + stream: TokenStream, + base_dir: Directory<'a>, +) -> Parser<'a> { + Parser::new(sess, stream, Some(base_dir), true, false, None) } /// A sequence separator. pub struct SeqSep { - /// The seperator token. - pub sep: Option<token::Token>, + /// The separator token. + pub sep: Option<TokenKind>, /// `true` if a trailing separator is allowed. pub trailing_sep_allowed: bool, } impl SeqSep { - pub fn trailing_allowed(t: token::Token) -> SeqSep { + pub fn trailing_allowed(t: TokenKind) -> SeqSep { SeqSep { sep: Some(t), trailing_sep_allowed: true, @@ -786,300 +392,3 @@ impl SeqSep { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::ast::{self, Ident, PatKind}; - use crate::attr::first_attr_value_str_by_name; - use crate::ptr::P; - use crate::print::pprust::item_to_string; - use crate::tokenstream::{DelimSpan, TokenTree}; - use crate::util::parser_testing::string_to_stream; - use crate::util::parser_testing::{string_to_expr, string_to_item}; - use crate::with_globals; - use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; - - /// Parses an item. - /// - /// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` - /// when a syntax error occurred. - fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<'_, Option<P<ast::Item>>> { - new_parser_from_source_str(sess, name, source).parse_item() - } - - // produce a syntax_pos::span - fn sp(a: u32, b: u32) -> Span { - Span::new(BytePos(a), BytePos(b), NO_EXPANSION) - } - - #[should_panic] - #[test] fn bad_path_expr_1() { - with_globals(|| { - string_to_expr("::abc::def::return".to_string()); - }) - } - - // check the token-tree-ization of macros - #[test] - fn string_to_tts_macro () { - with_globals(|| { - let tts: Vec<_> = - string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); - let tts: &[TokenTree] = &tts[..]; - - match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) { - ( - 4, - Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))), - Some(&TokenTree::Token(_, token::Not)), - Some(&TokenTree::Token(_, token::Ident(name_zip, false))), - Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)), - ) - if name_macro_rules.name == "macro_rules" - && name_zip.name == "zip" => { - let tts = ¯o_tts.trees().collect::<Vec<_>>(); - match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) { - ( - 3, - Some(&TokenTree::Delimited(_, first_delim, ref first_tts)), - Some(&TokenTree::Token(_, token::FatArrow)), - Some(&TokenTree::Delimited(_, second_delim, ref second_tts)), - ) - if macro_delim == token::Paren => { - let tts = &first_tts.trees().collect::<Vec<_>>(); - match (tts.len(), tts.get(0), tts.get(1)) { - ( - 2, - Some(&TokenTree::Token(_, token::Dollar)), - Some(&TokenTree::Token(_, token::Ident(ident, false))), - ) - if first_delim == token::Paren && ident.name == "a" => {}, - _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), - } - let tts = &second_tts.trees().collect::<Vec<_>>(); - match (tts.len(), tts.get(0), tts.get(1)) { - ( - 2, - Some(&TokenTree::Token(_, token::Dollar)), - Some(&TokenTree::Token(_, token::Ident(ident, false))), - ) - if second_delim == token::Paren && ident.name == "a" => {}, - _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), - } - }, - _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), - } - }, - _ => panic!("value: {:?}",tts), - } - }) - } - - #[test] - fn string_to_tts_1() { - with_globals(|| { - let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - - let expected = TokenStream::new(vec![ - TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), - TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(5, 6), sp(13, 14)), - token::DelimToken::Paren, - TokenStream::new(vec![ - TokenTree::Token(sp(6, 7), - token::Ident(Ident::from_str("b"), false)).into(), - TokenTree::Token(sp(8, 9), token::Colon).into(), - TokenTree::Token(sp(10, 13), - token::Ident(Ident::from_str("i32"), false)).into(), - ]).into(), - ).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(15, 16), sp(20, 21)), - token::DelimToken::Brace, - TokenStream::new(vec![ - TokenTree::Token(sp(17, 18), - token::Ident(Ident::from_str("b"), false)).into(), - TokenTree::Token(sp(18, 19), token::Semi).into(), - ]).into(), - ).into() - ]); - - assert_eq!(tts, expected); - }) - } - - #[test] fn parse_use() { - with_globals(|| { - let use_s = "use foo::bar::baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - - let use_s = "use foo::bar as baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - }) - } - - #[test] fn parse_extern_crate() { - with_globals(|| { - let ex_s = "extern crate foo;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - - let ex_s = "extern crate foo as bar;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - }) - } - - fn get_spans_of_pat_idents(src: &str) -> Vec<Span> { - let item = string_to_item(src.to_string()).unwrap(); - - struct PatIdentVisitor { - spans: Vec<Span> - } - impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { - fn visit_pat(&mut self, p: &'a ast::Pat) { - match p.node { - PatKind::Ident(_ , ref spannedident, _) => { - self.spans.push(spannedident.span.clone()); - } - _ => { - crate::visit::walk_pat(self, p); - } - } - } - } - let mut v = PatIdentVisitor { spans: Vec::new() }; - crate::visit::walk_item(&mut v, &item); - return v.spans; - } - - #[test] fn span_of_self_arg_pat_idents_are_correct() { - with_globals(|| { - - let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", - "impl z { fn a (&mut self, &myarg: i32) {} }", - "impl z { fn a (&'a self, &myarg: i32) {} }", - "impl z { fn a (self, &myarg: i32) {} }", - "impl z { fn a (self: Foo, &myarg: i32) {} }", - ]; - - for &src in &srcs { - let spans = get_spans_of_pat_idents(src); - let (lo, hi) = (spans[0].lo(), spans[0].hi()); - assert!("self" == &src[lo.to_usize()..hi.to_usize()], - "\"{}\" != \"self\". src=\"{}\"", - &src[lo.to_usize()..hi.to_usize()], src) - } - }) - } - - #[test] fn parse_exprs () { - with_globals(|| { - // just make sure that they parse.... - string_to_expr("3 + 4".to_string()); - string_to_expr("a::z.froob(b,&(987+3))".to_string()); - }) - } - - #[test] fn attrs_fix_bug () { - with_globals(|| { - string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) - -> Result<Box<Writer>, String> { - #[cfg(windows)] - fn wb() -> c_int { - (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int - } - - #[cfg(unix)] - fn wb() -> c_int { O_WRONLY as c_int } - - let mut fflags: c_int = wb(); -}".to_string()); - }) - } - - #[test] fn crlf_doc_comments() { - with_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - - let name_1 = FileName::Custom("crlf_source_1".to_string()); - let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_1, source, &sess) - .unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); - assert_eq!(doc, "/// doc comment"); - - let name_2 = FileName::Custom("crlf_source_2".to_string()); - let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_2, source, &sess) - .unwrap().unwrap(); - let docs = item.attrs.iter().filter(|a| a.path == "doc") - .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>(); - let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; - assert_eq!(&docs[..], b); - - let name_3 = FileName::Custom("clrf_source_3".to_string()); - let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap(); - assert_eq!(doc, "/** doc comment\n * with CRLF */"); - }); - } - - #[test] - fn ttdelim_span() { - fn parse_expr_from_source_str( - name: FileName, source: String, sess: &ParseSess - ) -> PResult<'_, P<ast::Expr>> { - new_parser_from_source_str(sess, name, source).parse_expr() - } - - with_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), - "foo!( fn main() { body } )".to_string(), &sess).unwrap(); - - let tts: Vec<_> = match expr.node { - ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(), - _ => panic!("not a macro"), - }; - - let span = tts.iter().rev().next().unwrap().span(); - - match sess.source_map().span_to_snippet(span) { - Ok(s) => assert_eq!(&s[..], "{ body }"), - Err(_) => panic!("could not get snippet"), - } - }); - } - - // This tests that when parsing a string (rather than a file) we don't try - // and read in a file for a module declaration and just parse a stub. - // See `recurse_into_file_modules` in the parser. - #[test] - fn out_of_line_mod() { - with_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let item = parse_item_from_source_str( - PathBuf::from("foo").into(), - "mod foo { struct S; mod this_does_not_exist; }".to_owned(), - &sess, - ).unwrap().unwrap(); - - if let ast::ItemKind::Mod(ref m) = item.node { - assert!(m.items.len() == 2); - } else { - panic!(); - } - }); - } -} diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index fe31311094b..d85c2df16a3 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -1,7 +1,9 @@ +// ignore-tidy-filelength + use crate::ast::{AngleBracketedArgs, ParenthesizedArgs, AttrStyle, BareFnTy}; use crate::ast::{GenericBound, TraitBoundModifier}; use crate::ast::Unsafety; -use crate::ast::{Mod, AnonConst, Arg, Arm, Guard, Attribute, BindingMode, TraitItemKind}; +use crate::ast::{Mod, AnonConst, Arg, Arm, Attribute, BindingMode, TraitItemKind}; use crate::ast::Block; use crate::ast::{BlockCheckMode, CaptureBy, Movability}; use crate::ast::{Constness, Crate}; @@ -13,7 +15,7 @@ use crate::ast::{ForeignItem, ForeignItemKind, FunctionRetTy}; use crate::ast::{GenericParam, GenericParamKind}; use crate::ast::GenericArg; use crate::ast::{Ident, ImplItem, IsAsync, IsAuto, Item, ItemKind}; -use crate::ast::{Label, Lifetime, Lit, LitKind}; +use crate::ast::{Label, Lifetime}; use crate::ast::Local; use crate::ast::MacStmtStyle; use crate::ast::{Mac, Mac_, MacDelimiter}; @@ -25,31 +27,33 @@ use crate::ast::{VariantData, StructField}; use crate::ast::StrStyle; use crate::ast::SelfKind; use crate::ast::{TraitItem, TraitRef, TraitObjectSyntax}; -use crate::ast::{Ty, TyKind, TypeBinding, GenericBounds}; +use crate::ast::{Ty, TyKind, AssocTyConstraint, AssocTyConstraintKind, GenericBounds}; use crate::ast::{Visibility, VisibilityKind, WhereClause, CrateSugar}; use crate::ast::{UseTree, UseTreeKind}; use crate::ast::{BinOpKind, UnOp}; use crate::ast::{RangeEnd, RangeSyntax}; use crate::{ast, attr}; use crate::ext::base::DummyResult; +use crate::ext::hygiene::SyntaxContext; use crate::source_map::{self, SourceMap, Spanned, respan}; -use crate::parse::{self, SeqSep, classify, token}; -use crate::parse::lexer::{TokenAndSpan, UnmatchedBrace}; +use crate::parse::{SeqSep, classify, literal, token}; +use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::DelimToken; +use crate::parse::token::{Token, TokenKind, DelimToken}; use crate::parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; -use crate::util::parser::{AssocOp, Fixity}; +use crate::util::parser::{AssocOp, Fixity, prec_let_scrutinee_needs_par}; use crate::print::pprust; use crate::ptr::P; use crate::parse::PResult; use crate::ThinVec; use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; -use crate::symbol::{Symbol, keywords}; +use crate::symbol::{kw, sym, Symbol}; +use crate::parse::diagnostics::{Error, dummy_arg}; use errors::{Applicability, DiagnosticBuilder, DiagnosticId, FatalError}; use rustc_target::spec::abi::{self, Abi}; -use syntax_pos::{Span, MultiSpan, BytePos, FileName}; -use log::{debug, trace}; +use syntax_pos::{Span, BytePos, DUMMY_SP, FileName}; +use log::debug; use std::borrow::Cow; use std::cmp; @@ -58,12 +62,12 @@ use std::path::{self, Path, PathBuf}; use std::slice; #[derive(Debug)] -/// Whether the type alias or associated type is a concrete type or an existential type +/// Whether the type alias or associated type is a concrete type or an opaque type pub enum AliasKind { /// Just a new name for the same type Weak(P<Ty>), /// Only trait impls of the type will be usable, not the actual type itself - Existential(GenericBounds), + OpaqueTy(GenericBounds), } bitflags::bitflags! { @@ -99,14 +103,14 @@ pub enum PathStyle { } #[derive(Clone, Copy, PartialEq, Debug)] -enum SemiColonMode { +crate enum SemiColonMode { Break, Ignore, Comma, } #[derive(Clone, Copy, PartialEq, Debug)] -enum BlockMode { +crate enum BlockMode { Break, Ignore, } @@ -118,24 +122,28 @@ enum BlockMode { /// `token::Interpolated` tokens. macro_rules! maybe_whole_expr { ($p:expr) => { - if let token::Interpolated(nt) = $p.token.clone() { - match *nt { - token::NtExpr(ref e) | token::NtLiteral(ref e) => { + if let token::Interpolated(nt) = &$p.token.kind { + match &**nt { + token::NtExpr(e) | token::NtLiteral(e) => { + let e = e.clone(); $p.bump(); - return Ok((*e).clone()); + return Ok(e); } - token::NtPath(ref path) => { + token::NtPath(path) => { + let path = path.clone(); $p.bump(); - let span = $p.span; - let kind = ExprKind::Path(None, (*path).clone()); - return Ok($p.mk_expr(span, kind, ThinVec::new())); + return Ok($p.mk_expr( + $p.token.span, ExprKind::Path(None, path), ThinVec::new() + )); } - token::NtBlock(ref block) => { + token::NtBlock(block) => { + let block = block.clone(); $p.bump(); - let span = $p.span; - let kind = ExprKind::Block((*block).clone(), None); - return Ok($p.mk_expr(span, kind, ThinVec::new())); + return Ok($p.mk_expr( + $p.token.span, ExprKind::Block(block, None), ThinVec::new() + )); } + // N.B: `NtIdent(ident)` is normalized to `Ident` in `fn bump`. _ => {}, }; } @@ -145,8 +153,9 @@ macro_rules! maybe_whole_expr { /// As maybe_whole_expr, but for things other than expressions macro_rules! maybe_whole { ($p:expr, $constructor:ident, |$x:ident| $e:expr) => { - if let token::Interpolated(nt) = $p.token.clone() { - if let token::$constructor($x) = (*nt).clone() { + if let token::Interpolated(nt) = &$p.token.kind { + if let token::$constructor(x) = &**nt { + let $x = x.clone(); $p.bump(); return Ok($e); } @@ -154,6 +163,21 @@ macro_rules! maybe_whole { }; } +/// If the next tokens are ill-formed `$ty::` recover them as `<$ty>::`. +macro_rules! maybe_recover_from_interpolated_ty_qpath { + ($self: expr, $allow_qpath_recovery: expr) => { + if $allow_qpath_recovery && $self.look_ahead(1, |t| t == &token::ModSep) { + if let token::Interpolated(nt) = &$self.token.kind { + if let token::NtTy(ty) = &**nt { + let ty = ty.clone(); + $self.bump(); + return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_span, ty); + } + } + } + } +} + fn maybe_append(mut lhs: Vec<Attribute>, mut rhs: Option<Vec<Attribute>>) -> Vec<Attribute> { if let Some(ref mut rhs) = rhs { lhs.append(rhs); @@ -169,81 +193,40 @@ enum PrevTokenKind { Interpolated, Eof, Ident, + BitOr, Other, } -trait RecoverQPath: Sized { - const PATH_STYLE: PathStyle = PathStyle::Expr; - fn to_ty(&self) -> Option<P<Ty>>; - fn to_recovered(&self, qself: Option<QSelf>, path: ast::Path) -> Self; - fn to_string(&self) -> String; -} - -impl RecoverQPath for Ty { - const PATH_STYLE: PathStyle = PathStyle::Type; - fn to_ty(&self) -> Option<P<Ty>> { - Some(P(self.clone())) - } - fn to_recovered(&self, qself: Option<QSelf>, path: ast::Path) -> Self { - Self { span: path.span, node: TyKind::Path(qself, path), id: self.id } - } - fn to_string(&self) -> String { - pprust::ty_to_string(self) - } -} - -impl RecoverQPath for Pat { - fn to_ty(&self) -> Option<P<Ty>> { - self.to_ty() - } - fn to_recovered(&self, qself: Option<QSelf>, path: ast::Path) -> Self { - Self { span: path.span, node: PatKind::Path(qself, path), id: self.id } - } - fn to_string(&self) -> String { - pprust::pat_to_string(self) - } -} - -impl RecoverQPath for Expr { - fn to_ty(&self) -> Option<P<Ty>> { - self.to_ty() - } - fn to_recovered(&self, qself: Option<QSelf>, path: ast::Path) -> Self { - Self { span: path.span, node: ExprKind::Path(qself, path), - id: self.id, attrs: self.attrs.clone() } - } - fn to_string(&self) -> String { - pprust::expr_to_string(self) - } -} - -/* ident is handled by common.rs */ +// NOTE: `Ident`s are handled by `common.rs`. #[derive(Clone)] pub struct Parser<'a> { pub sess: &'a ParseSess, - /// the current token: - pub token: token::Token, - /// the span of the current token: - pub span: Span, - /// the span of the previous token: + /// The current normalized token. + /// "Normalized" means that some interpolated tokens + /// (`$i: ident` and `$l: lifetime` meta-variables) are replaced + /// with non-interpolated identifier and lifetime tokens they refer to. + /// Perhaps the normalized / non-normalized setup can be simplified somehow. + pub token: Token, + /// Span of the current non-normalized token. meta_var_span: Option<Span>, + /// Span of the previous non-normalized token. pub prev_span: Span, - /// the previous token kind + /// Kind of the previous normalized token (in simplified form). prev_token_kind: PrevTokenKind, restrictions: Restrictions, - /// Used to determine the path to externally loaded source files + /// Used to determine the path to externally loaded source files. crate directory: Directory<'a>, - /// Whether to parse sub-modules in other files. + /// `true` to parse sub-modules in other files. pub recurse_into_file_modules: bool, /// Name of the root module this parser originated from. If `None`, then the /// name is not known. This does not change while the parser is descending /// into modules, and sub-parsers have new values for this name. pub root_module_name: Option<String>, crate expected_tokens: Vec<TokenType>, - token_cursor: TokenCursor, + crate token_cursor: TokenCursor, desugar_doc_comments: bool, - /// Whether we should configure out of line modules as we parse. + /// `true` we should configure out of line modules as we parse. pub cfg_mods: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error @@ -256,7 +239,10 @@ pub struct Parser<'a> { /// it gets removed from here. Every entry left at the end gets emitted as an independent /// error. crate unclosed_delims: Vec<UnmatchedBrace>, - last_unexpected_token_span: Option<Span>, + crate last_unexpected_token_span: Option<Span>, + crate last_type_ascription: Option<(Span, bool /* likely path typo */)>, + /// If present, this `Parser` is not parsing Rust code but rather a macro call. + crate subparser_name: Option<&'static str>, } impl<'a> Drop for Parser<'a> { @@ -267,19 +253,19 @@ impl<'a> Drop for Parser<'a> { } #[derive(Clone)] -struct TokenCursor { - frame: TokenCursorFrame, - stack: Vec<TokenCursorFrame>, +crate struct TokenCursor { + crate frame: TokenCursorFrame, + crate stack: Vec<TokenCursorFrame>, } #[derive(Clone)] -struct TokenCursorFrame { - delim: token::DelimToken, - span: DelimSpan, - open_delim: bool, - tree_cursor: tokenstream::Cursor, - close_delim: bool, - last_token: LastToken, +crate struct TokenCursorFrame { + crate delim: token::DelimToken, + crate span: DelimSpan, + crate open_delim: bool, + crate tree_cursor: tokenstream::Cursor, + crate close_delim: bool, + crate last_token: LastToken, } /// This is used in `TokenCursorFrame` above to track tokens that are consumed @@ -300,16 +286,16 @@ struct TokenCursorFrame { /// You can find some more example usage of this in the `collect_tokens` method /// on the parser. #[derive(Clone)] -enum LastToken { +crate enum LastToken { Collecting(Vec<TreeAndJoint>), Was(Option<TreeAndJoint>), } impl TokenCursorFrame { - fn new(sp: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { + fn new(span: DelimSpan, delim: DelimToken, tts: &TokenStream) -> Self { TokenCursorFrame { - delim: delim, - span: sp, + delim, + span, open_delim: delim == token::NoDelim, tree_cursor: tts.clone().into_trees(), close_delim: delim == token::NoDelim, @@ -319,7 +305,7 @@ impl TokenCursorFrame { } impl TokenCursor { - fn next(&mut self) -> TokenAndSpan { + fn next(&mut self) -> Token { loop { let tree = if !self.frame.open_delim { self.frame.open_delim = true; @@ -333,7 +319,7 @@ impl TokenCursor { self.frame = frame; continue } else { - return TokenAndSpan { tok: token::Eof, sp: syntax_pos::DUMMY_SP } + return Token::new(token::Eof, DUMMY_SP); }; match self.frame.last_token { @@ -342,7 +328,7 @@ impl TokenCursor { } match tree { - TokenTree::Token(sp, tok) => return TokenAndSpan { tok: tok, sp: sp }, + TokenTree::Token(token) => return token, TokenTree::Delimited(sp, delim, tts) => { let frame = TokenCursorFrame::new(sp, delim, &tts); self.stack.push(mem::replace(&mut self.frame, frame)); @@ -351,9 +337,9 @@ impl TokenCursor { } } - fn next_desugared(&mut self) -> TokenAndSpan { - let (sp, name) = match self.next() { - TokenAndSpan { sp, tok: token::DocComment(name) } => (sp, name), + fn next_desugared(&mut self) -> Token { + let (name, sp) = match self.next() { + Token { kind: token::DocComment(name), span } => (name, span), tok => return tok, }; @@ -376,10 +362,12 @@ impl TokenCursor { let body = TokenTree::Delimited( delim_span, token::Bracket, - [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)), - TokenTree::Token(sp, token::Eq), - TokenTree::Token(sp, token::Literal( - token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None)) + [ + TokenTree::token(token::Ident(sym::doc, false), sp), + TokenTree::token(token::Eq, sp), + TokenTree::token(TokenKind::lit( + token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None + ), sp), ] .iter().cloned().collect::<TokenStream>().into(), ); @@ -388,10 +376,10 @@ impl TokenCursor { delim_span, token::NoDelim, &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { - [TokenTree::Token(sp, token::Pound), TokenTree::Token(sp, token::Not), body] + [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] .iter().cloned().collect::<TokenStream>().into() } else { - [TokenTree::Token(sp, token::Pound), body] + [TokenTree::token(token::Pound, sp), body] .iter().cloned().collect::<TokenStream>().into() }, ))); @@ -402,8 +390,8 @@ impl TokenCursor { #[derive(Clone, PartialEq)] crate enum TokenType { - Token(token::Token), - Keyword(keywords::Keyword), + Token(TokenKind), + Keyword(Symbol), Operator, Lifetime, Ident, @@ -413,10 +401,10 @@ crate enum TokenType { } impl TokenType { - fn to_string(&self) -> String { + crate fn to_string(&self) -> String { match *self { - TokenType::Token(ref t) => format!("`{}`", pprust::token_to_string(t)), - TokenType::Keyword(kw) => format!("`{}`", kw.name()), + TokenType::Token(ref t) => format!("`{}`", pprust::token_kind_to_string(t)), + TokenType::Keyword(kw) => format!("`{}`", kw), TokenType::Operator => "an operator".to_string(), TokenType::Lifetime => "lifetime".to_string(), TokenType::Ident => "identifier".to_string(), @@ -432,7 +420,7 @@ impl TokenType { /// /// Types can also be of the form `IDENT(u8, u8) -> u8`, however this assumes /// that `IDENT` is not the ident of a fn trait. -fn can_continue_type_after_non_fn_ident(t: &token::Token) -> bool { +fn can_continue_type_after_non_fn_ident(t: &Token) -> bool { t == &token::ModSep || t == &token::Lt || t == &token::BinOp(token::Shl) } @@ -450,65 +438,6 @@ pub struct ModulePathSuccess { warn: bool, } -pub enum Error { - FileNotFoundForModule { - mod_name: String, - default_path: String, - secondary_path: String, - dir_path: String, - }, - DuplicatePaths { - mod_name: String, - default_path: String, - secondary_path: String, - }, - UselessDocComment, - InclusiveRangeWithNoEnd, -} - -impl Error { - fn span_err<S: Into<MultiSpan>>(self, - sp: S, - handler: &errors::Handler) -> DiagnosticBuilder<'_> { - match self { - Error::FileNotFoundForModule { ref mod_name, - ref default_path, - ref secondary_path, - ref dir_path } => { - let mut err = struct_span_err!(handler, sp, E0583, - "file not found for module `{}`", mod_name); - err.help(&format!("name the file either {} or {} inside the directory \"{}\"", - default_path, - secondary_path, - dir_path)); - err - } - Error::DuplicatePaths { ref mod_name, ref default_path, ref secondary_path } => { - let mut err = struct_span_err!(handler, sp, E0584, - "file for module `{}` found at both {} and {}", - mod_name, - default_path, - secondary_path); - err.help("delete or rename one of them to remove the ambiguity"); - err - } - Error::UselessDocComment => { - let mut err = struct_span_err!(handler, sp, E0585, - "found a documentation comment that doesn't document anything"); - err.help("doc comments must come before what they document, maybe a comment was \ - intended with `//`?"); - err - } - Error::InclusiveRangeWithNoEnd => { - let mut err = struct_span_err!(handler, sp, E0586, - "inclusive range with no end"); - err.help("inclusive ranges must be bounded at the end (`..=b` or `a..=b`)"); - err - } - } - } -} - #[derive(Debug)] enum LhsExpr { NotYetParsed, @@ -532,40 +461,25 @@ impl From<P<Expr>> for LhsExpr { } } -/// Creates a placeholder argument. -fn dummy_arg(span: Span) -> Arg { - let ident = Ident::new(keywords::Invalid.name(), span); - let pat = P(Pat { - id: ast::DUMMY_NODE_ID, - node: PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None), - span, - }); - let ty = Ty { - node: TyKind::Err, - span, - id: ast::DUMMY_NODE_ID - }; - Arg { ty: P(ty), pat: pat, id: ast::DUMMY_NODE_ID } -} - #[derive(Copy, Clone, Debug)] -enum TokenExpectType { +crate enum TokenExpectType { Expect, NoExpect, } impl<'a> Parser<'a> { - pub fn new(sess: &'a ParseSess, - tokens: TokenStream, - directory: Option<Directory<'a>>, - recurse_into_file_modules: bool, - desugar_doc_comments: bool) - -> Self { + pub fn new( + sess: &'a ParseSess, + tokens: TokenStream, + directory: Option<Directory<'a>>, + recurse_into_file_modules: bool, + desugar_doc_comments: bool, + subparser_name: Option<&'static str>, + ) -> Self { let mut parser = Parser { sess, - token: token::Whitespace, - span: syntax_pos::DUMMY_SP, - prev_span: syntax_pos::DUMMY_SP, + token: Token::dummy(), + prev_span: DUMMY_SP, meta_var_span: None, prev_token_kind: PrevTokenKind::Other, restrictions: Restrictions::empty(), @@ -590,16 +504,17 @@ impl<'a> Parser<'a> { max_angle_bracket_count: 0, unclosed_delims: Vec::new(), last_unexpected_token_span: None, + last_type_ascription: None, + subparser_name, }; - let tok = parser.next_tok(); - parser.token = tok.tok; - parser.span = tok.sp; + parser.token = parser.next_tok(); if let Some(directory) = directory { parser.directory = directory; - } else if !parser.span.is_dummy() { - if let FileName::Real(mut path) = sess.source_map().span_to_unmapped_path(parser.span) { + } else if !parser.token.span.is_dummy() { + if let FileName::Real(mut path) = + sess.source_map().span_to_unmapped_path(parser.token.span) { path.pop(); parser.directory.path = Cow::from(path); } @@ -609,15 +524,15 @@ impl<'a> Parser<'a> { parser } - fn next_tok(&mut self) -> TokenAndSpan { + fn next_tok(&mut self) -> Token { let mut next = if self.desugar_doc_comments { self.token_cursor.next_desugared() } else { self.token_cursor.next() }; - if next.sp.is_dummy() { + if next.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact. - next.sp = self.prev_span.with_ctxt(next.sp.ctxt()); + next.span = self.prev_span.with_ctxt(next.span.ctxt()); } next } @@ -627,17 +542,17 @@ impl<'a> Parser<'a> { pprust::token_to_string(&self.token) } - fn token_descr(&self) -> Option<&'static str> { - Some(match &self.token { - t if t.is_special_ident() => "reserved identifier", - t if t.is_used_keyword() => "keyword", - t if t.is_unused_keyword() => "reserved keyword", + crate fn token_descr(&self) -> Option<&'static str> { + Some(match &self.token.kind { + _ if self.token.is_special_ident() => "reserved identifier", + _ if self.token.is_used_keyword() => "keyword", + _ if self.token.is_unused_keyword() => "reserved keyword", token::DocComment(..) => "doc comment", _ => return None, }) } - fn this_token_descr(&self) -> String { + crate fn this_token_descr(&self) -> String { if let Some(prefix) = self.token_descr() { format!("{} `{}`", prefix, self.this_token_to_string()) } else { @@ -645,11 +560,6 @@ impl<'a> Parser<'a> { } } - fn unexpected_last<T>(&self, t: &token::Token) -> PResult<'a, T> { - let token_str = pprust::token_to_string(t); - Err(self.span_fatal(self.prev_span, &format!("unexpected token: `{}`", token_str))) - } - crate fn unexpected<T>(&mut self) -> PResult<'a, T> { match self.expect_one_of(&[], &[]) { Err(e) => Err(e), @@ -658,229 +568,45 @@ impl<'a> Parser<'a> { } /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. - pub fn expect(&mut self, t: &token::Token) -> PResult<'a, bool /* recovered */> { + pub fn expect(&mut self, t: &TokenKind) -> PResult<'a, bool /* recovered */> { if self.expected_tokens.is_empty() { if self.token == *t { self.bump(); Ok(false) } else { - let token_str = pprust::token_to_string(t); - let this_token_str = self.this_token_descr(); - let mut err = self.fatal(&format!("expected `{}`, found {}", - token_str, - this_token_str)); - - let sp = if self.token == token::Token::Eof { - // EOF, don't want to point at the following char, but rather the last token - self.prev_span - } else { - self.sess.source_map().next_point(self.prev_span) - }; - let label_exp = format!("expected `{}`", token_str); - match self.recover_closing_delimiter(&[t.clone()], err) { - Err(e) => err = e, - Ok(recovered) => { - return Ok(recovered); - } - } - let cm = self.sess.source_map(); - match (cm.lookup_line(self.span.lo()), cm.lookup_line(sp.lo())) { - (Ok(ref a), Ok(ref b)) if a.line == b.line => { - // When the spans are in the same line, it means that the only content - // between them is whitespace, point only at the found token. - err.span_label(self.span, label_exp); - } - _ => { - err.span_label(sp, label_exp); - err.span_label(self.span, "unexpected token"); - } - } - Err(err) + self.unexpected_try_recover(t) } } else { self.expect_one_of(slice::from_ref(t), &[]) } } - fn recover_closing_delimiter( - &mut self, - tokens: &[token::Token], - mut err: DiagnosticBuilder<'a>, - ) -> PResult<'a, bool> { - let mut pos = None; - // we want to use the last closing delim that would apply - for (i, unmatched) in self.unclosed_delims.iter().enumerate().rev() { - if tokens.contains(&token::CloseDelim(unmatched.expected_delim)) - && Some(self.span) > unmatched.unclosed_span - { - pos = Some(i); - } - } - match pos { - Some(pos) => { - // Recover and assume that the detected unclosed delimiter was meant for - // this location. Emit the diagnostic and act as if the delimiter was - // present for the parser's sake. - - // Don't attempt to recover from this unclosed delimiter more than once. - let unmatched = self.unclosed_delims.remove(pos); - let delim = TokenType::Token(token::CloseDelim(unmatched.expected_delim)); - - // We want to suggest the inclusion of the closing delimiter where it makes - // the most sense, which is immediately after the last token: - // - // {foo(bar {}} - // - ^ - // | | - // | help: `)` may belong here (FIXME: #58270) - // | - // unclosed delimiter - if let Some(sp) = unmatched.unclosed_span { - err.span_label(sp, "unclosed delimiter"); - } - err.span_suggestion_short( - self.sess.source_map().next_point(self.prev_span), - &format!("{} may belong here", delim.to_string()), - delim.to_string(), - Applicability::MaybeIncorrect, - ); - err.emit(); - self.expected_tokens.clear(); // reduce errors - Ok(true) - } - _ => Err(err), - } - } - /// Expect next token to be edible or inedible token. If edible, /// then consume it; if inedible, then return without consuming /// anything. Signal a fatal error if next token is unexpected. pub fn expect_one_of( &mut self, - edible: &[token::Token], - inedible: &[token::Token], + edible: &[TokenKind], + inedible: &[TokenKind], ) -> PResult<'a, bool /* recovered */> { - fn tokens_to_string(tokens: &[TokenType]) -> String { - let mut i = tokens.iter(); - // This might be a sign we need a connect method on Iterator. - let b = i.next() - .map_or(String::new(), |t| t.to_string()); - i.enumerate().fold(b, |mut b, (i, a)| { - if tokens.len() > 2 && i == tokens.len() - 2 { - b.push_str(", or "); - } else if tokens.len() == 2 && i == tokens.len() - 2 { - b.push_str(" or "); - } else { - b.push_str(", "); - } - b.push_str(&a.to_string()); - b - }) - } - if edible.contains(&self.token) { + if edible.contains(&self.token.kind) { self.bump(); Ok(false) - } else if inedible.contains(&self.token) { + } else if inedible.contains(&self.token.kind) { // leave it in the input Ok(false) - } else if self.last_unexpected_token_span == Some(self.span) { + } else if self.last_unexpected_token_span == Some(self.token.span) { FatalError.raise(); } else { - let mut expected = edible.iter() - .map(|x| TokenType::Token(x.clone())) - .chain(inedible.iter().map(|x| TokenType::Token(x.clone()))) - .chain(self.expected_tokens.iter().cloned()) - .collect::<Vec<_>>(); - expected.sort_by_cached_key(|x| x.to_string()); - expected.dedup(); - let expect = tokens_to_string(&expected[..]); - let actual = self.this_token_to_string(); - let (msg_exp, (label_sp, label_exp)) = if expected.len() > 1 { - let short_expect = if expected.len() > 6 { - format!("{} possible tokens", expected.len()) - } else { - expect.clone() - }; - (format!("expected one of {}, found `{}`", expect, actual), - (self.sess.source_map().next_point(self.prev_span), - format!("expected one of {} here", short_expect))) - } else if expected.is_empty() { - (format!("unexpected token: `{}`", actual), - (self.prev_span, "unexpected token after this".to_string())) - } else { - (format!("expected {}, found `{}`", expect, actual), - (self.sess.source_map().next_point(self.prev_span), - format!("expected {} here", expect))) - }; - self.last_unexpected_token_span = Some(self.span); - let mut err = self.fatal(&msg_exp); - if self.token.is_ident_named("and") { - err.span_suggestion_short( - self.span, - "use `&&` instead of `and` for the boolean operator", - "&&".to_string(), - Applicability::MaybeIncorrect, - ); - } - if self.token.is_ident_named("or") { - err.span_suggestion_short( - self.span, - "use `||` instead of `or` for the boolean operator", - "||".to_string(), - Applicability::MaybeIncorrect, - ); - } - let sp = if self.token == token::Token::Eof { - // This is EOF, don't want to point at the following char, but rather the last token - self.prev_span - } else { - label_sp - }; - match self.recover_closing_delimiter(&expected.iter().filter_map(|tt| match tt { - TokenType::Token(t) => Some(t.clone()), - _ => None, - }).collect::<Vec<_>>(), err) { - Err(e) => err = e, - Ok(recovered) => { - return Ok(recovered); - } - } - - let cm = self.sess.source_map(); - match (cm.lookup_line(self.span.lo()), cm.lookup_line(sp.lo())) { - (Ok(ref a), Ok(ref b)) if a.line == b.line => { - // When the spans are in the same line, it means that the only content between - // them is whitespace, point at the found token in that case: - // - // X | () => { syntax error }; - // | ^^^^^ expected one of 8 possible tokens here - // - // instead of having: - // - // X | () => { syntax error }; - // | -^^^^^ unexpected token - // | | - // | expected one of 8 possible tokens here - err.span_label(self.span, label_exp); - } - _ if self.prev_span == syntax_pos::DUMMY_SP => { - // Account for macro context where the previous span might not be - // available to avoid incorrect output (#54841). - err.span_label(self.span, "unexpected token"); - } - _ => { - err.span_label(sp, label_exp); - err.span_label(self.span, "unexpected token"); - } - } - Err(err) + self.expected_one_of_not_found(edible, inedible) } } /// Returns the span of expr, if it was not interpolated or the span of the interpolated token. - fn interpolated_or_expr_span(&self, - expr: PResult<'a, P<Expr>>) - -> PResult<'a, (Span, P<Expr>)> { + fn interpolated_or_expr_span( + &self, + expr: PResult<'a, P<Expr>>, + ) -> PResult<'a, (Span, P<Expr>)> { expr.map(|e| { if self.prev_token_kind == PrevTokenKind::Interpolated { (self.prev_span, e) @@ -890,45 +616,13 @@ impl<'a> Parser<'a> { }) } - fn expected_ident_found(&self) -> DiagnosticBuilder<'a> { - let mut err = self.struct_span_err(self.span, - &format!("expected identifier, found {}", - self.this_token_descr())); - if let token::Ident(ident, false) = &self.token { - if ident.is_reserved() && !ident.is_path_segment_keyword() && - ident.name != keywords::Underscore.name() - { - err.span_suggestion( - self.span, - "you can escape reserved keywords to use them as identifiers", - format!("r#{}", ident), - Applicability::MaybeIncorrect, - ); - } - } - if let Some(token_descr) = self.token_descr() { - err.span_label(self.span, format!("expected identifier, found {}", token_descr)); - } else { - err.span_label(self.span, "expected identifier"); - if self.token == token::Comma && self.look_ahead(1, |t| t.is_ident()) { - err.span_suggestion( - self.span, - "remove this comma", - String::new(), - Applicability::MachineApplicable, - ); - } - } - err - } - pub fn parse_ident(&mut self) -> PResult<'a, ast::Ident> { self.parse_ident_common(true) } fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> { - match self.token { - token::Ident(ident, _) => { + match self.token.kind { + token::Ident(name, _) => { if self.token.is_reserved_ident() { let mut err = self.expected_ident_found(); if recover { @@ -937,16 +631,16 @@ impl<'a> Parser<'a> { return Err(err); } } - let span = self.span; + let span = self.token.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => { Err(if self.prev_token_kind == PrevTokenKind::DocComment { - self.span_fatal_err(self.prev_span, Error::UselessDocComment) - } else { - self.expected_ident_found() - }) + self.span_fatal_err(self.prev_span, Error::UselessDocComment) + } else { + self.expected_ident_found() + }) } } } @@ -955,27 +649,27 @@ impl<'a> Parser<'a> { /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. - crate fn check(&mut self, tok: &token::Token) -> bool { + crate fn check(&mut self, tok: &TokenKind) -> bool { let is_present = self.token == *tok; if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } is_present } /// Consumes a token 'tok' if it exists. Returns whether the given token was present. - pub fn eat(&mut self, tok: &token::Token) -> bool { + pub fn eat(&mut self, tok: &TokenKind) -> bool { let is_present = self.check(tok); if is_present { self.bump() } is_present } - fn check_keyword(&mut self, kw: keywords::Keyword) -> bool { + fn check_keyword(&mut self, kw: Symbol) -> bool { self.expected_tokens.push(TokenType::Keyword(kw)); self.token.is_keyword(kw) } /// If the next token is the given keyword, eats it and returns /// `true`. Otherwise, returns `false`. - pub fn eat_keyword(&mut self, kw: keywords::Keyword) -> bool { + pub fn eat_keyword(&mut self, kw: Symbol) -> bool { if self.check_keyword(kw) { self.bump(); true @@ -984,7 +678,7 @@ impl<'a> Parser<'a> { } } - fn eat_keyword_noexpect(&mut self, kw: keywords::Keyword) -> bool { + fn eat_keyword_noexpect(&mut self, kw: Symbol) -> bool { if self.token.is_keyword(kw) { self.bump(); true @@ -996,7 +690,7 @@ impl<'a> Parser<'a> { /// If the given word is not a keyword, signals an error. /// If the next token is not the given word, signals an error. /// Otherwise, eats it. - fn expect_keyword(&mut self, kw: keywords::Keyword) -> PResult<'a, ()> { + fn expect_keyword(&mut self, kw: Symbol) -> PResult<'a, ()> { if !self.eat_keyword(kw) { self.unexpected() } else { @@ -1004,7 +698,7 @@ impl<'a> Parser<'a> { } } - fn check_ident(&mut self) -> bool { + crate fn check_ident(&mut self) -> bool { if self.token.is_ident() { true } else { @@ -1047,13 +741,13 @@ impl<'a> Parser<'a> { /// See issue #47856 for an example of when this may occur. fn eat_plus(&mut self) -> bool { self.expected_tokens.push(TokenType::Token(token::BinOp(token::Plus))); - match self.token { + match self.token.kind { token::BinOp(token::Plus) => { self.bump(); true } token::BinOpEq(token::Plus) => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); self.bump_with(token::Eq, span); true } @@ -1078,13 +772,13 @@ impl<'a> Parser<'a> { /// `&` and continues. If an `&` is not seen, signals an error. fn expect_and(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::BinOp(token::And))); - match self.token { + match self.token.kind { token::BinOp(token::And) => { self.bump(); Ok(()) } token::AndAnd => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); Ok(self.bump_with(token::BinOp(token::And), span)) } _ => self.unexpected() @@ -1095,13 +789,13 @@ impl<'a> Parser<'a> { /// `|` and continues. If an `|` is not seen, signals an error. fn expect_or(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::BinOp(token::Or))); - match self.token { + match self.token.kind { token::BinOp(token::Or) => { self.bump(); Ok(()) } token::OrOr => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); Ok(self.bump_with(token::BinOp(token::Or), span)) } _ => self.unexpected() @@ -1109,19 +803,7 @@ impl<'a> Parser<'a> { } fn expect_no_suffix(&self, sp: Span, kind: &str, suffix: Option<ast::Name>) { - match suffix { - None => {/* everything ok */} - Some(suf) => { - let text = suf.as_str(); - if text.is_empty() { - self.span_bug(sp, "found empty literal suffix in Some") - } - let msg = format!("{} with a suffix is invalid", kind); - self.struct_span_err(sp, &msg) - .span_label(sp, msg) - .emit(); - } - } + literal::expect_no_suffix(&self.sess.span_diagnostic, sp, kind, suffix) } /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single @@ -1132,18 +814,18 @@ impl<'a> Parser<'a> { /// starting token. fn eat_lt(&mut self) -> bool { self.expected_tokens.push(TokenType::Token(token::Lt)); - let ate = match self.token { + let ate = match self.token.kind { token::Lt => { self.bump(); true } token::BinOp(token::Shl) => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); self.bump_with(token::Lt, span); true } token::LArrow => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); self.bump_with(token::BinOp(token::Minus), span); true } @@ -1172,21 +854,21 @@ impl<'a> Parser<'a> { /// with a single `>` and continues. If a `>` is not seen, signals an error. fn expect_gt(&mut self) -> PResult<'a, ()> { self.expected_tokens.push(TokenType::Token(token::Gt)); - let ate = match self.token { + let ate = match self.token.kind { token::Gt => { self.bump(); Some(()) } token::BinOp(token::Shr) => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); Some(self.bump_with(token::Gt, span)) } token::BinOpEq(token::Shr) => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); Some(self.bump_with(token::Ge, span)) } token::Ge => { - let span = self.span.with_lo(self.span.lo() + BytePos(1)); + let span = self.token.span.with_lo(self.token.span.lo() + BytePos(1)); Some(self.bump_with(token::Eq, span)) } _ => None, @@ -1206,30 +888,16 @@ impl<'a> Parser<'a> { } } - /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, - /// passes through any errors encountered. Used for error recovery. - fn eat_to_tokens(&mut self, kets: &[&token::Token]) { - let handler = self.diagnostic(); - - if let Err(ref mut err) = self.parse_seq_to_before_tokens(kets, - SeqSep::none(), - TokenExpectType::Expect, - |p| Ok(p.parse_token_tree())) { - handler.cancel(err); - } - } - /// Parses a sequence, including the closing delimiter. The function /// `f` must consume tokens until reaching the next separator or /// closing bracket. - pub fn parse_seq_to_end<T, F>(&mut self, - ket: &token::Token, - sep: SeqSep, - f: F) - -> PResult<'a, Vec<T>> where - F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, - { - let (val, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + pub fn parse_seq_to_end<T>( + &mut self, + ket: &TokenKind, + sep: SeqSep, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, Vec<T>> { + let (val, _, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; if !recovered { self.bump(); } @@ -1239,39 +907,39 @@ impl<'a> Parser<'a> { /// Parses a sequence, not including the closing delimiter. The function /// `f` must consume tokens until reaching the next separator or /// closing bracket. - pub fn parse_seq_to_before_end<T, F>( + pub fn parse_seq_to_before_end<T>( &mut self, - ket: &token::Token, + ket: &TokenKind, sep: SeqSep, - f: F, - ) -> PResult<'a, (Vec<T>, bool)> - where F: FnMut(&mut Parser<'a>) -> PResult<'a, T> - { + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec<T>, bool, bool)> { self.parse_seq_to_before_tokens(&[ket], sep, TokenExpectType::Expect, f) } - fn parse_seq_to_before_tokens<T, F>( + fn expect_any_with_type(&mut self, kets: &[&TokenKind], expect: TokenExpectType) -> bool { + kets.iter().any(|k| { + match expect { + TokenExpectType::Expect => self.check(k), + TokenExpectType::NoExpect => self.token == **k, + } + }) + } + + crate fn parse_seq_to_before_tokens<T>( &mut self, - kets: &[&token::Token], + kets: &[&TokenKind], sep: SeqSep, expect: TokenExpectType, - mut f: F, - ) -> PResult<'a, (Vec<T>, bool /* recovered */)> - where F: FnMut(&mut Parser<'a>) -> PResult<'a, T> - { + mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec<T>, bool /* trailing */, bool /* recovered */)> { let mut first = true; let mut recovered = false; + let mut trailing = false; let mut v = vec![]; - while !kets.iter().any(|k| { - match expect { - TokenExpectType::Expect => self.check(k), - TokenExpectType::NoExpect => self.token == **k, - } - }) { - match self.token { - token::CloseDelim(..) | token::Eof => break, - _ => {} - }; + while !self.expect_any_with_type(kets, expect) { + if let token::CloseDelim(..) | token::Eof = self.token.kind { + break + } if let Some(ref t) = sep.sep { if first { first = false; @@ -1285,7 +953,7 @@ impl<'a> Parser<'a> { Err(mut e) => { // Attempt to keep parsing if it was a similar separator if let Some(ref tokens) = t.similar_tokens() { - if tokens.contains(&self.token) { + if tokens.contains(&self.token.kind) { self.bump(); } } @@ -1305,12 +973,8 @@ impl<'a> Parser<'a> { } } } - if sep.trailing_sep_allowed && kets.iter().any(|k| { - match expect { - TokenExpectType::Expect => self.check(k), - TokenExpectType::NoExpect => self.token == **k, - } - }) { + if sep.trailing_sep_allowed && self.expect_any_with_type(kets, expect) { + trailing = true; break; } @@ -1318,27 +982,45 @@ impl<'a> Parser<'a> { v.push(t); } - Ok((v, recovered)) + Ok((v, trailing, recovered)) } /// Parses a sequence, including the closing delimiter. The function /// `f` must consume tokens until reaching the next separator or /// closing bracket. - fn parse_unspanned_seq<T, F>( + fn parse_unspanned_seq<T>( &mut self, - bra: &token::Token, - ket: &token::Token, + bra: &TokenKind, + ket: &TokenKind, sep: SeqSep, - f: F, - ) -> PResult<'a, Vec<T>> where - F: FnMut(&mut Parser<'a>) -> PResult<'a, T>, - { + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec<T>, bool)> { self.expect(bra)?; - let (result, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; + let (result, trailing, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; if !recovered { self.eat(ket); } - Ok(result) + Ok((result, trailing)) + } + + fn parse_delim_comma_seq<T>( + &mut self, + delim: DelimToken, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec<T>, bool)> { + self.parse_unspanned_seq( + &token::OpenDelim(delim), + &token::CloseDelim(delim), + SeqSep::trailing_allowed(token::Comma), + f, + ) + } + + fn parse_paren_comma_seq<T>( + &mut self, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> PResult<'a, (Vec<T>, bool)> { + self.parse_delim_comma_seq(token::Paren, f) } /// Advance the parser by one token @@ -1348,22 +1030,21 @@ impl<'a> Parser<'a> { self.bug("attempted to bump the parser past EOF (may be stuck in a loop)"); } - self.prev_span = self.meta_var_span.take().unwrap_or(self.span); + self.prev_span = self.meta_var_span.take().unwrap_or(self.token.span); // Record last token kind for possible error recovery. - self.prev_token_kind = match self.token { + self.prev_token_kind = match self.token.kind { token::DocComment(..) => PrevTokenKind::DocComment, token::Comma => PrevTokenKind::Comma, token::BinOp(token::Plus) => PrevTokenKind::Plus, + token::BinOp(token::Or) => PrevTokenKind::BitOr, token::Interpolated(..) => PrevTokenKind::Interpolated, token::Eof => PrevTokenKind::Eof, token::Ident(..) => PrevTokenKind::Ident, _ => PrevTokenKind::Other, }; - let next = self.next_tok(); - self.span = next.sp; - self.token = next.tok; + self.token = self.next_tok(); self.expected_tokens.clear(); // check after each token self.process_potential_macro_variable(); @@ -1371,79 +1052,44 @@ impl<'a> Parser<'a> { /// Advance the parser using provided token as a next one. Use this when /// consuming a part of a token. For example a single `<` from `<<`. - fn bump_with(&mut self, next: token::Token, span: Span) { - self.prev_span = self.span.with_hi(span.lo()); + fn bump_with(&mut self, next: TokenKind, span: Span) { + self.prev_span = self.token.span.with_hi(span.lo()); // It would be incorrect to record the kind of the current token, but // fortunately for tokens currently using `bump_with`, the // prev_token_kind will be of no use anyway. self.prev_token_kind = PrevTokenKind::Other; - self.span = span; - self.token = next; + self.token = Token::new(next, span); self.expected_tokens.clear(); } pub fn look_ahead<R, F>(&self, dist: usize, f: F) -> R where - F: FnOnce(&token::Token) -> R, + F: FnOnce(&Token) -> R, { if dist == 0 { - return f(&self.token) + return f(&self.token); } - f(&match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { + let frame = &self.token_cursor.frame; + f(&match frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { - TokenTree::Token(_, tok) => tok, - TokenTree::Delimited(_, delim, _) => token::OpenDelim(delim), - }, - None => token::CloseDelim(self.token_cursor.frame.delim), + TokenTree::Token(token) => token, + TokenTree::Delimited(dspan, delim, _) => + Token::new(token::OpenDelim(delim), dspan.open), + } + None => Token::new(token::CloseDelim(frame.delim), frame.span.close) }) } - fn look_ahead_span(&self, dist: usize) -> Span { - if dist == 0 { - return self.span - } - - match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { - Some(TokenTree::Token(span, _)) => span, - Some(TokenTree::Delimited(span, ..)) => span.entire(), - None => self.look_ahead_span(dist - 1), - } - } - pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> { - self.sess.span_diagnostic.struct_span_fatal(self.span, m) - } - pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { - self.sess.span_diagnostic.struct_span_fatal(sp, m) - } - fn span_fatal_err<S: Into<MultiSpan>>(&self, sp: S, err: Error) -> DiagnosticBuilder<'a> { - err.span_err(sp, self.diagnostic()) - } - fn bug(&self, m: &str) -> ! { - self.sess.span_diagnostic.span_bug(self.span, m) - } - fn span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) { - self.sess.span_diagnostic.span_err(sp, m) - } - fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> { - self.sess.span_diagnostic.struct_span_err(sp, m) - } - crate fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! { - self.sess.span_diagnostic.span_bug(sp, m) - } - - fn cancel(&self, err: &mut DiagnosticBuilder<'_>) { - self.sess.span_diagnostic.cancel(err) - } - - crate fn diagnostic(&self) -> &'a errors::Handler { - &self.sess.span_diagnostic + /// Returns whether any of the given keywords are `dist` tokens ahead of the current one. + fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool { + self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw))) } /// Is the current token one of the keywords that signals a bare function type? fn token_is_bare_fn_keyword(&mut self) -> bool { - self.check_keyword(keywords::Fn) || - self.check_keyword(keywords::Unsafe) || - self.check_keyword(keywords::Extern) + self.check_keyword(kw::Fn) || + self.check_keyword(kw::Unsafe) || + self.check_keyword(kw::Extern) } /// Parses a `TyKind::BareFn` type. @@ -1461,13 +1107,13 @@ impl<'a> Parser<'a> { */ let unsafety = self.parse_unsafety(); - let abi = if self.eat_keyword(keywords::Extern) { + let abi = if self.eat_keyword(kw::Extern) { self.parse_opt_abi()?.unwrap_or(Abi::C) } else { Abi::Rust }; - self.expect_keyword(keywords::Fn)?; + self.expect_keyword(kw::Fn)?; let (inputs, c_variadic) = self.parse_fn_args(false, true)?; let ret_ty = self.parse_ret_ty(false)?; let decl = P(FnDecl { @@ -1485,7 +1131,7 @@ impl<'a> Parser<'a> { /// Parses asyncness: `async` or nothing. fn parse_asyncness(&mut self) -> IsAsync { - if self.eat_keyword(keywords::Async) { + if self.eat_keyword(kw::Async) { IsAsync::Async { closure_id: ast::DUMMY_NODE_ID, return_impl_trait_id: ast::DUMMY_NODE_ID, @@ -1497,7 +1143,7 @@ impl<'a> Parser<'a> { /// Parses unsafety: `unsafe` or nothing. fn parse_unsafety(&mut self) -> Unsafety { - if self.eat_keyword(keywords::Unsafe) { + if self.eat_keyword(kw::Unsafe) { Unsafety::Unsafe } else { Unsafety::Normal @@ -1525,12 +1171,12 @@ impl<'a> Parser<'a> { fn parse_trait_item_(&mut self, at_end: &mut bool, mut attrs: Vec<Attribute>) -> PResult<'a, TraitItem> { - let lo = self.span; - - let (name, node, generics) = if self.eat_keyword(keywords::Type) { + let lo = self.token.span; + self.eat_bad_pub(); + let (name, node, generics) = if self.eat_keyword(kw::Type) { self.parse_trait_item_assoc_ty()? } else if self.is_const_item() { - self.expect_keyword(keywords::Const)?; + self.expect_keyword(kw::Const)?; let ident = self.parse_ident()?; self.expect(&token::Colon)?; let ty = self.parse_ty()?; @@ -1545,20 +1191,21 @@ impl<'a> Parser<'a> { (ident, TraitItemKind::Const(ty, default), ast::Generics::default()) } else if let Some(mac) = self.parse_assoc_macro_invoc("trait", None, &mut false)? { // trait item macro. - (keywords::Invalid.ident(), ast::TraitItemKind::Macro(mac), ast::Generics::default()) + (Ident::invalid(), ast::TraitItemKind::Macro(mac), ast::Generics::default()) } else { let (constness, unsafety, asyncness, abi) = self.parse_fn_front_matter()?; let ident = self.parse_ident()?; let mut generics = self.parse_generics()?; - let d = self.parse_fn_decl_with_self(|p: &mut Parser<'a>| { + let decl = self.parse_fn_decl_with_self(|p: &mut Parser<'a>| { // This is somewhat dubious; We don't want to allow // argument names to be left off if there is a // definition... // We don't allow argument names to be left off in edition 2018. - p.parse_arg_general(p.span.rust_2018(), true, false) + let is_name_required = p.token.span.rust_2018(); + p.parse_arg_general(true, false, |_| is_name_required) })?; generics.where_clause = self.parse_where_clause()?; @@ -1569,10 +1216,10 @@ impl<'a> Parser<'a> { abi, asyncness, }, - decl: d, + decl, }; - let body = match self.token { + let body = match self.token.kind { token::Semi => { self.bump(); *at_end = true; @@ -1595,20 +1242,12 @@ impl<'a> Parser<'a> { Some(body) } _ => { - let token_str = self.this_token_descr(); - let mut err = self.fatal(&format!("expected `;` or `{{`, found {}", - token_str)); - err.span_label(self.span, "expected `;` or `{`"); - return Err(err); + return self.expected_semi_or_open_brace(); } } } _ => { - let token_str = self.this_token_descr(); - let mut err = self.fatal(&format!("expected `;` or `{{`, found {}", - token_str)); - err.span_label(self.span, "expected `;` or `{`"); - return Err(err); + return self.expected_semi_or_open_brace(); } }; (ident, ast::TraitItemKind::Method(sig, body), generics) @@ -1630,7 +1269,7 @@ impl<'a> Parser<'a> { if self.eat(&token::RArrow) { Ok(FunctionRetTy::Ty(self.parse_ty_common(allow_plus, true, false)?)) } else { - Ok(FunctionRetTy::Default(self.span.shrink_to_lo())) + Ok(FunctionRetTy::Default(self.token.span.shrink_to_lo())) } } @@ -1651,9 +1290,10 @@ impl<'a> Parser<'a> { fn parse_ty_common(&mut self, allow_plus: bool, allow_qpath_recovery: bool, allow_c_variadic: bool) -> PResult<'a, P<Ty>> { + maybe_recover_from_interpolated_ty_qpath!(self, allow_qpath_recovery); maybe_whole!(self, NtTy, |x| x); - let lo = self.span; + let lo = self.token.span; let mut impl_dyn_multi = false; let node = if self.eat(&token::OpenDelim(token::Paren)) { // `(TYPE)` is a parenthesized type. @@ -1717,7 +1357,7 @@ impl<'a> Parser<'a> { // Reference self.expect_and()?; self.parse_borrowed_pointee()? - } else if self.eat_keyword_noexpect(keywords::Typeof) { + } else if self.eat_keyword_noexpect(kw::Typeof) { // `typeof(EXPR)` // In order to not be ambiguous, the type must be surrounded by parens. self.expect(&token::OpenDelim(token::Paren))?; @@ -1727,17 +1367,17 @@ impl<'a> Parser<'a> { }; self.expect(&token::CloseDelim(token::Paren))?; TyKind::Typeof(e) - } else if self.eat_keyword(keywords::Underscore) { + } else if self.eat_keyword(kw::Underscore) { // A type to be inferred `_` TyKind::Infer } else if self.token_is_bare_fn_keyword() { // Function pointer type self.parse_ty_bare_fn(Vec::new())? - } else if self.check_keyword(keywords::For) { + } else if self.check_keyword(kw::For) { // Function pointer type or bound list (trait object type) starting with a poly-trait. // `for<'lt> [unsafe] [extern "ABI"] fn (&'lt S) -> T` // `for<'lt> Trait1<'lt> + Trait2 + 'a` - let lo = self.span; + let lo = self.token.span; let lifetime_defs = self.parse_late_bound_lifetime_defs()?; if self.token_is_bare_fn_keyword() { self.parse_ty_bare_fn(lifetime_defs)? @@ -1746,13 +1386,13 @@ impl<'a> Parser<'a> { let parse_plus = allow_plus && self.check_plus(); self.parse_remaining_bounds(lifetime_defs, path, lo, parse_plus)? } - } else if self.eat_keyword(keywords::Impl) { + } else if self.eat_keyword(kw::Impl) { // Always parse bounds greedily for better error recovery. let bounds = self.parse_generic_bounds(None)?; impl_dyn_multi = bounds.len() > 1 || self.prev_token_kind == PrevTokenKind::Plus; TyKind::ImplTrait(ast::DUMMY_NODE_ID, bounds) - } else if self.check_keyword(keywords::Dyn) && - (self.span.rust_2018() || + } else if self.check_keyword(kw::Dyn) && + (self.token.span.rust_2018() || self.look_ahead(1, |t| t.can_begin_bound() && !can_continue_type_after_non_fn_ident(t))) { self.bump(); // `dyn` @@ -1775,7 +1415,12 @@ impl<'a> Parser<'a> { if self.eat(&token::Not) { // Macro invocation in type position let (delim, tts) = self.expect_delimited_token_tree()?; - let node = Mac_ { path, tts, delim }; + let node = Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + }; TyKind::Mac(respan(lo.to(self.prev_span), node)) } else { // Just a type path or bound list (trait object type) starting with a trait. @@ -1798,18 +1443,19 @@ impl<'a> Parser<'a> { } } else { let msg = format!("expected type, found {}", self.this_token_descr()); - return Err(self.fatal(&msg)); + let mut err = self.fatal(&msg); + err.span_label(self.token.span, "expected type"); + self.maybe_annotate_with_ascription(&mut err, true); + return Err(err); }; let span = lo.to(self.prev_span); - let ty = Ty { node, span, id: ast::DUMMY_NODE_ID }; + let ty = P(Ty { node, span, id: ast::DUMMY_NODE_ID }); // Try to recover from use of `+` with incorrect priority. self.maybe_report_ambiguous_plus(allow_plus, impl_dyn_multi, &ty); self.maybe_recover_from_bad_type_plus(allow_plus, &ty)?; - let ty = self.maybe_recover_from_bad_qpath(ty, allow_qpath_recovery)?; - - Ok(P(ty)) + self.maybe_recover_from_bad_qpath(ty, allow_qpath_recovery) } fn parse_remaining_bounds(&mut self, generic_params: Vec<GenericParam>, path: ast::Path, @@ -1823,106 +1469,17 @@ impl<'a> Parser<'a> { Ok(TyKind::TraitObject(bounds, TraitObjectSyntax::None)) } - fn maybe_report_ambiguous_plus(&mut self, allow_plus: bool, impl_dyn_multi: bool, ty: &Ty) { - if !allow_plus && impl_dyn_multi { - let sum_with_parens = format!("({})", pprust::ty_to_string(&ty)); - self.struct_span_err(ty.span, "ambiguous `+` in a type") - .span_suggestion( - ty.span, - "use parentheses to disambiguate", - sum_with_parens, - Applicability::MachineApplicable - ).emit(); - } - } - - fn maybe_recover_from_bad_type_plus(&mut self, allow_plus: bool, ty: &Ty) -> PResult<'a, ()> { - // Do not add `+` to expected tokens. - if !allow_plus || !self.token.is_like_plus() { - return Ok(()) - } - - self.bump(); // `+` - let bounds = self.parse_generic_bounds(None)?; - let sum_span = ty.span.to(self.prev_span); - - let mut err = struct_span_err!(self.sess.span_diagnostic, sum_span, E0178, - "expected a path on the left-hand side of `+`, not `{}`", pprust::ty_to_string(ty)); - - match ty.node { - TyKind::Rptr(ref lifetime, ref mut_ty) => { - let sum_with_parens = pprust::to_string(|s| { - use crate::print::pprust::PrintState; - - s.s.word("&")?; - s.print_opt_lifetime(lifetime)?; - s.print_mutability(mut_ty.mutbl)?; - s.popen()?; - s.print_type(&mut_ty.ty)?; - s.print_type_bounds(" +", &bounds)?; - s.pclose() - }); - err.span_suggestion( - sum_span, - "try adding parentheses", - sum_with_parens, - Applicability::MachineApplicable - ); - } - TyKind::Ptr(..) | TyKind::BareFn(..) => { - err.span_label(sum_span, "perhaps you forgot parentheses?"); - } - _ => { - err.span_label(sum_span, "expected a path"); - }, - } - err.emit(); - Ok(()) - } - - // Try to recover from associated item paths like `[T]::AssocItem`/`(T, U)::AssocItem`. - fn maybe_recover_from_bad_qpath<T: RecoverQPath>(&mut self, base: T, allow_recovery: bool) - -> PResult<'a, T> { - // Do not add `::` to expected tokens. - if !allow_recovery || self.token != token::ModSep { - return Ok(base); - } - let ty = match base.to_ty() { - Some(ty) => ty, - None => return Ok(base), - }; - - self.bump(); // `::` - let mut segments = Vec::new(); - self.parse_path_segments(&mut segments, T::PATH_STYLE, true)?; - - let span = ty.span.to(self.prev_span); - let path_span = span.to(span); // use an empty path since `position` == 0 - let recovered = base.to_recovered( - Some(QSelf { ty, path_span, position: 0 }), - ast::Path { segments, span }, - ); - - self.diagnostic() - .struct_span_err(span, "missing angle brackets in associated item path") - .span_suggestion( // this is a best-effort recovery - span, "try", recovered.to_string(), Applicability::MaybeIncorrect - ).emit(); - - Ok(recovered) - } - fn parse_borrowed_pointee(&mut self) -> PResult<'a, TyKind> { let opt_lifetime = if self.check_lifetime() { Some(self.expect_lifetime()) } else { None }; let mutbl = self.parse_mutability(); let ty = self.parse_ty_no_plus()?; - return Ok(TyKind::Rptr(opt_lifetime, MutTy { ty: ty, mutbl: mutbl })); + return Ok(TyKind::Rptr(opt_lifetime, MutTy { ty, mutbl })); } fn parse_ptr(&mut self) -> PResult<'a, MutTy> { - let mutbl = if self.eat_keyword(keywords::Mut) { + let mutbl = if self.eat_keyword(kw::Mut) { Mutability::Mutable - } else if self.eat_keyword(keywords::Const) { + } else if self.eat_keyword(kw::Const) { Mutability::Immutable } else { let span = self.prev_span; @@ -1934,17 +1491,17 @@ impl<'a> Parser<'a> { Mutability::Immutable }; let t = self.parse_ty_no_plus()?; - Ok(MutTy { ty: t, mutbl: mutbl }) + Ok(MutTy { ty: t, mutbl }) } - fn is_named_argument(&mut self) -> bool { - let offset = match self.token { + fn is_named_argument(&self) -> bool { + let offset = match self.token.kind { token::Interpolated(ref nt) => match **nt { token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon), _ => 0, } token::BinOp(token::And) | token::AndAnd => 1, - _ if self.token.is_keyword(keywords::Mut) => 1, + _ if self.token.is_keyword(kw::Mut) => 1, _ => 0, }; @@ -1954,90 +1511,48 @@ impl<'a> Parser<'a> { /// Skips unexpected attributes and doc comments in this position and emits an appropriate /// error. - fn eat_incorrect_doc_comment(&mut self, applied_to: &str) { - if let token::DocComment(_) = self.token { - let mut err = self.diagnostic().struct_span_err( - self.span, - &format!("documentation comments cannot be applied to {}", applied_to), - ); - err.span_label(self.span, "doc comments are not allowed here"); - err.emit(); - self.bump(); - } else if self.token == token::Pound && self.look_ahead(1, |t| { - *t == token::OpenDelim(token::Bracket) - }) { - let lo = self.span; - // Skip every token until next possible arg. - while self.token != token::CloseDelim(token::Bracket) { - self.bump(); - } - let sp = lo.to(self.span); - self.bump(); - let mut err = self.diagnostic().struct_span_err( - sp, - &format!("attributes cannot be applied to {}", applied_to), - ); - err.span_label(sp, "attributes are not allowed here"); - err.emit(); - } - } - /// This version of parse arg doesn't necessarily require identifier names. - fn parse_arg_general(&mut self, require_name: bool, is_trait_item: bool, - allow_c_variadic: bool) -> PResult<'a, Arg> { - maybe_whole!(self, NtArg, |x| x); - - if let Ok(Some(_)) = self.parse_self_arg() { - let mut err = self.struct_span_err(self.prev_span, - "unexpected `self` argument in function"); - err.span_label(self.prev_span, - "`self` is only valid as the first argument of an associated function"); - return Err(err); + fn parse_arg_general<F>( + &mut self, + is_trait_item: bool, + allow_c_variadic: bool, + is_name_required: F, + ) -> PResult<'a, Arg> + where + F: Fn(&token::Token) -> bool + { + let lo = self.token.span; + let attrs = self.parse_arg_attributes()?; + if let Some(mut arg) = self.parse_self_arg()? { + arg.attrs = attrs.into(); + return self.recover_bad_self_arg(arg, is_trait_item); } - let (pat, ty) = if require_name || self.is_named_argument() { - debug!("parse_arg_general parse_pat (require_name:{})", - require_name); - self.eat_incorrect_doc_comment("method arguments"); - let pat = self.parse_pat(Some("argument name"))?; + let is_name_required = is_name_required(&self.token); + let (pat, ty) = if is_name_required || self.is_named_argument() { + debug!("parse_arg_general parse_pat (is_name_required:{})", is_name_required); + let pat = self.parse_pat(Some("argument name"))?; if let Err(mut err) = self.expect(&token::Colon) { - // If we find a pattern followed by an identifier, it could be an (incorrect) - // C-style parameter declaration. - if self.check_ident() && self.look_ahead(1, |t| { - *t == token::Comma || *t == token::CloseDelim(token::Paren) - }) { - let ident = self.parse_ident().unwrap(); - let span = pat.span.with_hi(ident.span.hi()); - - err.span_suggestion( - span, - "declare the type after the parameter binding", - String::from("<identifier>: <type>"), - Applicability::HasPlaceholders, - ); - } else if require_name && is_trait_item { - if let PatKind::Ident(_, ident, _) = pat.node { - err.span_suggestion( - pat.span, - "explicitly ignore parameter", - format!("_: {}", ident), - Applicability::MachineApplicable, - ); - } - - err.note("anonymous parameters are removed in the 2018 edition (see RFC 1685)"); + if let Some(ident) = self.argument_without_type( + &mut err, + pat, + is_name_required, + is_trait_item, + ) { + err.emit(); + return Ok(dummy_arg(ident)); + } else { + return Err(err); } - - return Err(err); } - self.eat_incorrect_doc_comment("a method argument's type"); + self.eat_incorrect_doc_comment_for_arg_type(); (pat, self.parse_ty_common(true, true, allow_c_variadic)?) } else { debug!("parse_arg_general ident_to_pat"); let parser_snapshot_before_ty = self.clone(); - self.eat_incorrect_doc_comment("a method argument's type"); + self.eat_incorrect_doc_comment_for_arg_type(); let mut ty = self.parse_ty_common(true, true, allow_c_variadic); if ty.is_ok() && self.token != token::Comma && self.token != token::CloseDelim(token::Paren) { @@ -2047,7 +1562,7 @@ impl<'a> Parser<'a> { } match ty { Ok(ty) => { - let ident = Ident::new(keywords::Invalid.name(), self.prev_span); + let ident = Ident::new(kw::Invalid, self.prev_span); let pat = P(Pat { id: ast::DUMMY_NODE_ID, node: PatKind::Ident( @@ -2065,44 +1580,20 @@ impl<'a> Parser<'a> { // Recover from attempting to parse the argument as a type without pattern. err.cancel(); mem::replace(self, parser_snapshot_before_ty); - let pat = self.parse_pat(Some("argument name"))?; - self.expect(&token::Colon)?; - let ty = self.parse_ty()?; - - let mut err = self.diagnostic().struct_span_err_with_code( - pat.span, - "patterns aren't allowed in methods without bodies", - DiagnosticId::Error("E0642".into()), - ); - err.span_suggestion_short( - pat.span, - "give this argument a name or use an underscore to ignore it", - "_".to_owned(), - Applicability::MachineApplicable, - ); - err.emit(); - - // Pretend the pattern is `_`, to avoid duplicate errors from AST validation. - let pat = P(Pat { - node: PatKind::Wild, - span: pat.span, - id: ast::DUMMY_NODE_ID - }); - (pat, ty) + self.recover_arg_parse()? } } }; - Ok(Arg { ty, pat, id: ast::DUMMY_NODE_ID }) - } + let span = lo.to(self.token.span); - /// Parses a single function argument. - crate fn parse_arg(&mut self) -> PResult<'a, Arg> { - self.parse_arg_general(true, false, false) + Ok(Arg { attrs: attrs.into(), id: ast::DUMMY_NODE_ID, pat, span, ty }) } /// Parses an argument in a lambda header (e.g., `|arg, arg|`). fn parse_fn_block_arg(&mut self) -> PResult<'a, Arg> { + let lo = self.token.span; + let attrs = self.parse_arg_attributes()?; let pat = self.parse_pat(Some("argument name"))?; let t = if self.eat(&token::Colon) { self.parse_ty()? @@ -2113,9 +1604,12 @@ impl<'a> Parser<'a> { span: self.prev_span, }) }; + let span = lo.to(self.token.span); Ok(Arg { + attrs: attrs.into(), ty: t, pat, + span, id: ast::DUMMY_NODE_ID }) } @@ -2128,93 +1622,13 @@ impl<'a> Parser<'a> { } } - /// Matches `token_lit = LIT_INTEGER | ...`. - fn parse_lit_token(&mut self) -> PResult<'a, LitKind> { - let out = match self.token { - token::Interpolated(ref nt) => match **nt { - token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node { - ExprKind::Lit(ref lit) => { lit.node.clone() } - _ => { return self.unexpected_last(&self.token); } - }, - _ => { return self.unexpected_last(&self.token); } - }, - token::Literal(lit, suf) => { - let diag = Some((self.span, &self.sess.span_diagnostic)); - let (suffix_illegal, result) = parse::lit_token(lit, suf, diag); - - if suffix_illegal { - let sp = self.span; - self.expect_no_suffix(sp, lit.literal_name(), suf) - } - - result.unwrap() - } - token::Dot if self.look_ahead(1, |t| match t { - token::Literal(parse::token::Lit::Integer(_) , _) => true, - _ => false, - }) => { // recover from `let x = .4;` - let lo = self.span; - self.bump(); - if let token::Literal( - parse::token::Lit::Integer(val), - suffix, - ) = self.token { - let suffix = suffix.and_then(|s| { - let s = s.as_str().get(); - if ["f32", "f64"].contains(&s) { - Some(s) - } else { - None - } - }).unwrap_or(""); - self.bump(); - let sp = lo.to(self.prev_span); - let mut err = self.diagnostic() - .struct_span_err(sp, "float literals must have an integer part"); - err.span_suggestion( - sp, - "must have an integer part", - format!("0.{}{}", val, suffix), - Applicability::MachineApplicable, - ); - err.emit(); - return Ok(match suffix { - "f32" => ast::LitKind::Float(val, ast::FloatTy::F32), - "f64" => ast::LitKind::Float(val, ast::FloatTy::F64), - _ => ast::LitKind::FloatUnsuffixed(val), - }); - } else { - unreachable!(); - }; - } - _ => { return self.unexpected_last(&self.token); } - }; - - self.bump(); - Ok(out) - } - - /// Matches `lit = true | false | token_lit`. - crate fn parse_lit(&mut self) -> PResult<'a, Lit> { - let lo = self.span; - let lit = if self.eat_keyword(keywords::True) { - LitKind::Bool(true) - } else if self.eat_keyword(keywords::False) { - LitKind::Bool(false) - } else { - let lit = self.parse_lit_token()?; - lit - }; - Ok(source_map::Spanned { node: lit, span: lo.to(self.prev_span) }) - } - /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`). crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> { maybe_whole_expr!(self); - let minus_lo = self.span; + let minus_lo = self.token.span; let minus_present = self.eat(&token::BinOp(token::Minus)); - let lo = self.span; + let lo = self.token.span; let literal = self.parse_lit()?; let hi = self.prev_span; let expr = self.mk_expr(lo.to(hi), ExprKind::Lit(literal), ThinVec::new()); @@ -2229,22 +1643,22 @@ impl<'a> Parser<'a> { } fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> { - match self.token { - token::Ident(ident, _) if self.token.is_path_segment_keyword() => { - let span = self.span; + match self.token.kind { + token::Ident(name, _) if name.is_path_segment_keyword() => { + let span = self.token.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => self.parse_ident(), } } fn parse_ident_or_underscore(&mut self) -> PResult<'a, ast::Ident> { - match self.token { - token::Ident(ident, false) if ident.name == keywords::Underscore.name() => { - let span = self.span; + match self.token.kind { + token::Ident(name, false) if name == kw::Underscore => { + let span = self.token.span; self.bump(); - Ok(Ident::new(ident.name, span)) + Ok(Ident::new(name, span)) } _ => self.parse_ident(), } @@ -2269,13 +1683,13 @@ impl<'a> Parser<'a> { // above). `path_span` has the span of that path, or an empty // span in the case of something like `<T>::Bar`. let (mut path, path_span); - if self.eat_keyword(keywords::As) { - let path_lo = self.span; + if self.eat_keyword(kw::As) { + let path_lo = self.token.span; path = self.parse_path(PathStyle::Type)?; path_span = path_lo.to(self.prev_span); } else { - path = ast::Path { segments: Vec::new(), span: syntax_pos::DUMMY_SP }; - path_span = self.span.to(self.span); + path_span = self.token.span.to(self.token.span); + path = ast::Path { segments: Vec::new(), span: path_span }; } // See doc comment for `unmatched_angle_bracket_count`. @@ -2288,7 +1702,7 @@ impl<'a> Parser<'a> { self.expect(&token::ModSep)?; let qself = QSelf { ty, path_span, position: path.segments.len() }; - self.parse_path_segments(&mut path.segments, style, true)?; + self.parse_path_segments(&mut path.segments, style)?; Ok((qself, ast::Path { segments: path.segments, span: lo.to(self.prev_span) })) } @@ -2304,11 +1718,6 @@ impl<'a> Parser<'a> { /// `Fn(Args)` (without disambiguator) /// `Fn::(Args)` (with disambiguator) pub fn parse_path(&mut self, style: PathStyle) -> PResult<'a, ast::Path> { - self.parse_path_common(style, true) - } - - crate fn parse_path_common(&mut self, style: PathStyle, enable_warning: bool) - -> PResult<'a, ast::Path> { maybe_whole!(self, NtPath, |path| { if style == PathStyle::Mod && path.segments.iter().any(|segment| segment.args.is_some()) { @@ -2317,13 +1726,13 @@ impl<'a> Parser<'a> { path }); - let lo = self.meta_var_span.unwrap_or(self.span); + let lo = self.meta_var_span.unwrap_or(self.token.span); let mut segments = Vec::new(); - let mod_sep_ctxt = self.span.ctxt(); + let mod_sep_ctxt = self.token.span.ctxt(); if self.eat(&token::ModSep) { segments.push(PathSegment::path_root(lo.shrink_to_lo().with_ctxt(mod_sep_ctxt))); } - self.parse_path_segments(&mut segments, style, enable_warning)?; + self.parse_path_segments(&mut segments, style)?; Ok(ast::Path { segments, span: lo.to(self.prev_span) }) } @@ -2332,10 +1741,10 @@ impl<'a> Parser<'a> { /// backwards-compatibility. This is used when parsing derive macro paths in `#[derive]` /// attributes. pub fn parse_path_allowing_meta(&mut self, style: PathStyle) -> PResult<'a, ast::Path> { - let meta_ident = match self.token { + let meta_ident = match self.token.kind { token::Interpolated(ref nt) => match **nt { token::NtMeta(ref meta) => match meta.node { - ast::MetaItemKind::Word => Some(meta.ident.clone()), + ast::MetaItemKind::Word => Some(meta.path.clone()), _ => None, }, _ => None, @@ -2349,13 +1758,12 @@ impl<'a> Parser<'a> { self.parse_path(style) } - fn parse_path_segments(&mut self, + crate fn parse_path_segments(&mut self, segments: &mut Vec<PathSegment>, - style: PathStyle, - enable_warning: bool) + style: PathStyle) -> PResult<'a, ()> { loop { - let segment = self.parse_path_segment(style, enable_warning)?; + let segment = self.parse_path_segment(style)?; if style == PathStyle::Expr { // In order to check for trailing angle brackets, we must have finished // recursing (`parse_path_segment` can indirectly call this function), @@ -2383,12 +1791,12 @@ impl<'a> Parser<'a> { } } - fn parse_path_segment(&mut self, style: PathStyle, enable_warning: bool) - -> PResult<'a, PathSegment> { + fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, PathSegment> { let ident = self.parse_path_segment_ident()?; - let is_args_start = |token: &token::Token| match *token { - token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren) => true, + let is_args_start = |token: &Token| match token.kind { + token::Lt | token::BinOp(token::Shl) | token::OpenDelim(token::Paren) + | token::LArrow => true, _ => false, }; let check_args_start = |this: &mut Self| { @@ -2401,13 +1809,6 @@ impl<'a> Parser<'a> { Ok(if style == PathStyle::Type && check_args_start(self) || style != PathStyle::Mod && self.check(&token::ModSep) && self.look_ahead(1, |t| is_args_start(t)) { - // Generic arguments are found - `<`, `(`, `::<` or `::(`. - if self.eat(&token::ModSep) && style == PathStyle::Type && enable_warning { - self.diagnostic().struct_span_warn(self.prev_span, "unnecessary path disambiguator") - .span_label(self.prev_span, "try removing `::`").emit(); - } - let lo = self.span; - // We use `style == PathStyle::Expr` to check if this is in a recursion or not. If // it isn't, then we reset the unmatched angle bracket count as we're about to start // parsing a new path. @@ -2416,24 +1817,19 @@ impl<'a> Parser<'a> { self.max_angle_bracket_count = 0; } + // Generic arguments are found - `<`, `(`, `::<` or `::(`. + self.eat(&token::ModSep); + let lo = self.token.span; let args = if self.eat_lt() { // `<'a, T, A = U>` - let (args, bindings) = + let (args, constraints) = self.parse_generic_args_with_leaning_angle_bracket_recovery(style, lo)?; self.expect_gt()?; let span = lo.to(self.prev_span); - AngleBracketedArgs { args, bindings, span }.into() + AngleBracketedArgs { args, constraints, span }.into() } else { // `(T, U) -> R` - self.bump(); // `(` - let (inputs, recovered) = self.parse_seq_to_before_tokens( - &[&token::CloseDelim(token::Paren)], - SeqSep::trailing_allowed(token::Comma), - TokenExpectType::Expect, - |p| p.parse_ty())?; - if !recovered { - self.bump(); // `)` - } + let (inputs, _) = self.parse_paren_comma_seq(|p| p.parse_ty())?; let span = lo.to(self.prev_span); let output = if self.eat(&token::RArrow) { Some(self.parse_ty_common(false, false, false)?) @@ -2458,17 +1854,17 @@ impl<'a> Parser<'a> { /// Parses a single lifetime `'a` or panics. crate fn expect_lifetime(&mut self) -> Lifetime { if let Some(ident) = self.token.lifetime() { - let span = self.span; + let span = self.token.span; self.bump(); Lifetime { ident: Ident::new(ident.name, span), id: ast::DUMMY_NODE_ID } } else { - self.span_bug(self.span, "not a lifetime") + self.span_bug(self.token.span, "not a lifetime") } } fn eat_label(&mut self) -> Option<Label> { if let Some(ident) = self.token.lifetime() { - let span = self.span; + let span = self.token.span; self.bump(); Some(Label { ident: Ident::new(ident.name, span) }) } else { @@ -2478,7 +1874,7 @@ impl<'a> Parser<'a> { /// Parses mutability (`mut` or nothing). fn parse_mutability(&mut self) -> Mutability { - if self.eat_keyword(keywords::Mut) { + if self.eat_keyword(kw::Mut) { Mutability::Mutable } else { Mutability::Immutable @@ -2486,9 +1882,11 @@ impl<'a> Parser<'a> { } fn parse_field_name(&mut self) -> PResult<'a, Ident> { - if let token::Literal(token::Integer(name), None) = self.token { + if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = + self.token.kind { + self.expect_no_suffix(self.token.span, "a tuple index", suffix); self.bump(); - Ok(Ident::new(name, self.prev_span)) + Ok(Ident::new(symbol, self.prev_span)) } else { self.parse_ident_common(false) } @@ -2497,7 +1895,7 @@ impl<'a> Parser<'a> { /// Parse ident (COLON expr)? fn parse_field(&mut self) -> PResult<'a, Field> { let attrs = self.parse_outer_attributes()?; - let lo = self.span; + let lo = self.token.span; // Check if a colon exists one ahead. This means we're parsing a fieldname. let (fieldname, expr, is_shorthand) = if self.look_ahead(1, |t| { @@ -2509,9 +1907,9 @@ impl<'a> Parser<'a> { // initialize a field with an eq rather than a colon. if self.token == token::Eq { self.diagnostic() - .struct_span_err(self.span, "expected `:`, found `=`") + .struct_span_err(self.token.span, "expected `:`, found `=`") .span_suggestion( - fieldname.span.shrink_to_hi().to(self.span), + fieldname.span.shrink_to_hi().to(self.token.span), "replace equals symbol with a colon", ":".to_string(), Applicability::MachineApplicable, @@ -2537,59 +1935,50 @@ impl<'a> Parser<'a> { }) } - fn mk_expr(&mut self, span: Span, node: ExprKind, attrs: ThinVec<Attribute>) -> P<Expr> { + crate fn mk_expr(&self, span: Span, node: ExprKind, attrs: ThinVec<Attribute>) -> P<Expr> { P(Expr { node, span, attrs, id: ast::DUMMY_NODE_ID }) } - fn mk_unary(&mut self, unop: ast::UnOp, expr: P<Expr>) -> ast::ExprKind { + fn mk_unary(&self, unop: ast::UnOp, expr: P<Expr>) -> ast::ExprKind { ExprKind::Unary(unop, expr) } - fn mk_binary(&mut self, binop: ast::BinOp, lhs: P<Expr>, rhs: P<Expr>) -> ast::ExprKind { + fn mk_binary(&self, binop: ast::BinOp, lhs: P<Expr>, rhs: P<Expr>) -> ast::ExprKind { ExprKind::Binary(binop, lhs, rhs) } - fn mk_call(&mut self, f: P<Expr>, args: Vec<P<Expr>>) -> ast::ExprKind { + fn mk_call(&self, f: P<Expr>, args: Vec<P<Expr>>) -> ast::ExprKind { ExprKind::Call(f, args) } - fn mk_index(&mut self, expr: P<Expr>, idx: P<Expr>) -> ast::ExprKind { + fn mk_index(&self, expr: P<Expr>, idx: P<Expr>) -> ast::ExprKind { ExprKind::Index(expr, idx) } - fn mk_range(&mut self, + fn mk_range(&self, start: Option<P<Expr>>, end: Option<P<Expr>>, limits: RangeLimits) -> PResult<'a, ast::ExprKind> { if end.is_none() && limits == RangeLimits::Closed { - Err(self.span_fatal_err(self.span, Error::InclusiveRangeWithNoEnd)) + Err(self.span_fatal_err(self.token.span, Error::InclusiveRangeWithNoEnd)) } else { Ok(ExprKind::Range(start, end, limits)) } } - fn mk_assign_op(&mut self, binop: ast::BinOp, + fn mk_assign_op(&self, binop: ast::BinOp, lhs: P<Expr>, rhs: P<Expr>) -> ast::ExprKind { ExprKind::AssignOp(binop, lhs, rhs) } - pub fn mk_mac_expr(&mut self, span: Span, m: Mac_, attrs: ThinVec<Attribute>) -> P<Expr> { - P(Expr { - id: ast::DUMMY_NODE_ID, - node: ExprKind::Mac(source_map::Spanned {node: m, span: span}), - span, - attrs, - }) - } - fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> { - let delim = match self.token { + let delim = match self.token.kind { token::OpenDelim(delim) => delim, _ => { let msg = "expected open delimiter"; let mut err = self.fatal(msg); - err.span_label(self.span, msg); + err.span_label(self.token.span, msg); return Err(err) } }; @@ -2612,6 +2001,7 @@ impl<'a> Parser<'a> { /// N.B., this does not parse outer attributes, and is private because it only works /// correctly if called from `parse_dot_or_call_expr()`. fn parse_bottom_expr(&mut self) -> PResult<'a, P<Expr>> { + maybe_recover_from_interpolated_ty_qpath!(self, true); maybe_whole_expr!(self); // Outer attributes are already parsed and will be @@ -2621,13 +2011,34 @@ impl<'a> Parser<'a> { // attributes by giving them a empty "already parsed" list. let mut attrs = ThinVec::new(); - let lo = self.span; - let mut hi = self.span; + let lo = self.token.span; + let mut hi = self.token.span; let ex: ExprKind; - // Note: when adding new syntax here, don't forget to adjust Token::can_begin_expr(). - match self.token { + macro_rules! parse_lit { + () => { + match self.parse_lit() { + Ok(literal) => { + hi = self.prev_span; + ex = ExprKind::Lit(literal); + } + Err(mut err) => { + self.cancel(&mut err); + return Err(self.expected_expression_found()); + } + } + } + } + + // Note: when adding new syntax here, don't forget to adjust TokenKind::can_begin_expr(). + match self.token.kind { + // This match arm is a special-case of the `_` match arm below and + // could be removed without changing functionality, but it's faster + // to have it here, especially for programs with large constants. + token::Literal(_) => { + parse_lit!() + } token::OpenDelim(token::Paren) => { self.bump(); @@ -2639,7 +2050,13 @@ impl<'a> Parser<'a> { let mut trailing_comma = false; let mut recovered = false; while self.token != token::CloseDelim(token::Paren) { - es.push(self.parse_expr()?); + es.push(match self.parse_expr() { + Ok(es) => es, + Err(err) => { + // recover from parse error in tuple list + return Ok(self.recover_seq_parse_error(token::Paren, lo, Err(err))); + } + }); recovered = self.expect_one_of( &[], &[token::Comma, token::CloseDelim(token::Paren)], @@ -2711,38 +2128,30 @@ impl<'a> Parser<'a> { hi = path.span; return Ok(self.mk_expr(lo.to(hi), ExprKind::Path(Some(qself), path), attrs)); } - if self.span.rust_2018() && self.check_keyword(keywords::Async) - { - if self.is_async_block() { // check for `async {` and `async move {` - return self.parse_async_block(attrs); - } else { - return self.parse_lambda_expr(attrs); - } - } - if self.check_keyword(keywords::Move) || self.check_keyword(keywords::Static) { + if self.check_keyword(kw::Move) || self.check_keyword(kw::Static) { return self.parse_lambda_expr(attrs); } - if self.eat_keyword(keywords::If) { + if self.eat_keyword(kw::If) { return self.parse_if_expr(attrs); } - if self.eat_keyword(keywords::For) { + if self.eat_keyword(kw::For) { let lo = self.prev_span; return self.parse_for_expr(None, lo, attrs); } - if self.eat_keyword(keywords::While) { + if self.eat_keyword(kw::While) { let lo = self.prev_span; return self.parse_while_expr(None, lo, attrs); } if let Some(label) = self.eat_label() { let lo = label.ident.span; self.expect(&token::Colon)?; - if self.eat_keyword(keywords::While) { + if self.eat_keyword(kw::While) { return self.parse_while_expr(Some(label), lo, attrs) } - if self.eat_keyword(keywords::For) { + if self.eat_keyword(kw::For) { return self.parse_for_expr(Some(label), lo, attrs) } - if self.eat_keyword(keywords::Loop) { + if self.eat_keyword(kw::Loop) { return self.parse_loop_expr(Some(label), lo, attrs) } if self.token == token::OpenDelim(token::Brace) { @@ -2753,27 +2162,27 @@ impl<'a> Parser<'a> { } let msg = "expected `while`, `for`, `loop` or `{` after a label"; let mut err = self.fatal(msg); - err.span_label(self.span, msg); + err.span_label(self.token.span, msg); return Err(err); } - if self.eat_keyword(keywords::Loop) { + if self.eat_keyword(kw::Loop) { let lo = self.prev_span; return self.parse_loop_expr(None, lo, attrs); } - if self.eat_keyword(keywords::Continue) { + if self.eat_keyword(kw::Continue) { let label = self.eat_label(); let ex = ExprKind::Continue(label); let hi = self.prev_span; return Ok(self.mk_expr(lo.to(hi), ex, attrs)); } - if self.eat_keyword(keywords::Match) { + if self.eat_keyword(kw::Match) { let match_sp = self.prev_span; return self.parse_match_expr(attrs).map_err(|mut err| { err.span_label(match_sp, "while parsing this match expression"); err }); } - if self.eat_keyword(keywords::Unsafe) { + if self.eat_keyword(kw::Unsafe) { return self.parse_block_expr( None, lo, @@ -2786,11 +2195,21 @@ impl<'a> Parser<'a> { return Err(db); } if self.is_try_block() { - let lo = self.span; - assert!(self.eat_keyword(keywords::Try)); + let lo = self.token.span; + assert!(self.eat_keyword(kw::Try)); return self.parse_try_block(lo, attrs); } - if self.eat_keyword(keywords::Return) { + + // Span::rust_2018() is somewhat expensive; don't get it repeatedly. + let is_span_rust_2018 = self.token.span.rust_2018(); + if is_span_rust_2018 && self.check_keyword(kw::Async) { + return if self.is_async_block() { // check for `async {` and `async move {` + self.parse_async_block(attrs) + } else { + self.parse_lambda_expr(attrs) + }; + } + if self.eat_keyword(kw::Return) { if self.token.can_begin_expr() { let e = self.parse_expr()?; hi = e.span; @@ -2798,7 +2217,7 @@ impl<'a> Parser<'a> { } else { ex = ExprKind::Ret(None); } - } else if self.eat_keyword(keywords::Break) { + } else if self.eat_keyword(kw::Break) { let label = self.eat_label(); let e = if self.token.can_begin_expr() && !(self.token == token::OpenDelim(token::Brace) @@ -2810,7 +2229,7 @@ impl<'a> Parser<'a> { }; ex = ExprKind::Break(label, e); hi = self.prev_span; - } else if self.eat_keyword(keywords::Yield) { + } else if self.eat_keyword(kw::Yield) { if self.token.can_begin_expr() { let e = self.parse_expr()?; hi = e.span; @@ -2818,37 +2237,37 @@ impl<'a> Parser<'a> { } else { ex = ExprKind::Yield(None); } - } else if self.token.is_keyword(keywords::Let) { - // Catch this syntax error here, instead of in `parse_ident`, so - // that we can explicitly mention that let is not to be used as an expression - let mut db = self.fatal("expected expression, found statement (`let`)"); - db.span_label(self.span, "expected expression"); - db.note("variable declaration using `let` is a statement"); - return Err(db); + } else if self.eat_keyword(kw::Let) { + return self.parse_let_expr(attrs); + } else if is_span_rust_2018 && self.eat_keyword(kw::Await) { + let (await_hi, e_kind) = self.parse_incorrect_await_syntax(lo, self.prev_span)?; + hi = await_hi; + ex = e_kind; } else if self.token.is_path_start() { - let pth = self.parse_path(PathStyle::Expr)?; + let path = self.parse_path(PathStyle::Expr)?; // `!`, as an operator, is prefix, so we know this isn't that if self.eat(&token::Not) { // MACRO INVOCATION expression let (delim, tts) = self.expect_delimited_token_tree()?; - let hi = self.prev_span; - let node = Mac_ { path: pth, tts, delim }; - return Ok(self.mk_mac_expr(lo.to(hi), node, attrs)) - } - if self.check(&token::OpenDelim(token::Brace)) { - // This is a struct literal, unless we're prohibited - // from parsing struct literals here. - let prohibited = self.restrictions.contains( - Restrictions::NO_STRUCT_LITERAL - ); - if !prohibited { - return self.parse_struct_expr(lo, pth, attrs); + hi = self.prev_span; + ex = ExprKind::Mac(respan(lo.to(hi), Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + })); + } else if self.check(&token::OpenDelim(token::Brace)) { + if let Some(expr) = self.maybe_parse_struct_expr(lo, &path, &attrs) { + return expr; + } else { + hi = path.span; + ex = ExprKind::Path(None, path); } + } else { + hi = path.span; + ex = ExprKind::Path(None, path); } - - hi = pth.span; - ex = ExprKind::Path(None, pth); } else { if !self.unclosed_delims.is_empty() && self.check(&token::Semi) { // Don't complain about bare semicolons after unclosed braces @@ -2863,30 +2282,56 @@ impl<'a> Parser<'a> { // | ^ expected expression // ``` self.bump(); - return Ok(self.mk_expr(self.span, ExprKind::Err, ThinVec::new())); - } - match self.parse_literal_maybe_minus() { - Ok(expr) => { - hi = expr.span; - ex = expr.node.clone(); - } - Err(mut err) => { - self.cancel(&mut err); - let msg = format!("expected expression, found {}", - self.this_token_descr()); - let mut err = self.fatal(&msg); - err.span_label(self.span, "expected expression"); - return Err(err); - } + return Ok(self.mk_expr(self.token.span, ExprKind::Err, ThinVec::new())); } + parse_lit!() } } } - let expr = Expr { node: ex, span: lo.to(hi), id: ast::DUMMY_NODE_ID, attrs }; - let expr = self.maybe_recover_from_bad_qpath(expr, true)?; + let expr = self.mk_expr(lo.to(hi), ex, attrs); + self.maybe_recover_from_bad_qpath(expr, true) + } - return Ok(P(expr)); + fn maybe_parse_struct_expr( + &mut self, + lo: Span, + path: &ast::Path, + attrs: &ThinVec<Attribute>, + ) -> Option<PResult<'a, P<Expr>>> { + let struct_allowed = !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL); + let certainly_not_a_block = || self.look_ahead(1, |t| t.is_ident()) && ( + // `{ ident, ` cannot start a block + self.look_ahead(2, |t| t == &token::Comma) || + self.look_ahead(2, |t| t == &token::Colon) && ( + // `{ ident: token, ` cannot start a block + self.look_ahead(4, |t| t == &token::Comma) || + // `{ ident: ` cannot start a block unless it's a type ascription `ident: Type` + self.look_ahead(3, |t| !t.can_begin_type()) + ) + ); + + if struct_allowed || certainly_not_a_block() { + // This is a struct literal, but we don't can't accept them here + let expr = self.parse_struct_expr(lo, path.clone(), attrs.clone()); + if let (Ok(expr), false) = (&expr, struct_allowed) { + self.struct_span_err( + expr.span, + "struct literals are not allowed here", + ) + .multipart_suggestion( + "surround the struct literal with parentheses", + vec![ + (lo.shrink_to_lo(), "(".to_string()), + (expr.span.shrink_to_hi(), ")".to_string()), + ], + Applicability::MachineApplicable, + ) + .emit(); + } + return Some(expr); + } + None } fn parse_struct_expr(&mut self, lo: Span, pth: ast::Path, mut attrs: ThinVec<Attribute>) @@ -2911,33 +2356,31 @@ impl<'a> Parser<'a> { } } if self.token == token::Comma { - let mut err = self.sess.span_diagnostic.mut_span_err( + self.struct_span_err( exp_span.to(self.prev_span), "cannot use a comma after the base struct", - ); - err.span_suggestion_short( - self.span, + ) + .span_suggestion_short( + self.token.span, "remove this comma", String::new(), Applicability::MachineApplicable - ); - err.note("the base struct must always be the last field"); - err.emit(); + ) + .note("the base struct must always be the last field") + .emit(); self.recover_stmt(); } break; } let mut recovery_field = None; - if let token::Ident(ident, _) = self.token { + if let token::Ident(name, _) = self.token.kind { if !self.token.is_reserved_ident() && self.look_ahead(1, |t| *t == token::Colon) { // Use in case of error after field-looking code: `S { foo: () with a }` - let mut ident = ident.clone(); - ident.span = self.span; recovery_field = Some(ast::Field { - ident, - span: self.span, - expr: self.mk_expr(self.span, ExprKind::Err, ThinVec::new()), + ident: Ident::new(name, self.token.span), + span: self.token.span, + expr: self.mk_expr(self.token.span, ExprKind::Err, ThinVec::new()), is_shorthand: false, attrs: ThinVec::new(), }); @@ -2980,7 +2423,7 @@ impl<'a> Parser<'a> { } } - let span = lo.to(self.span); + let span = lo.to(self.token.span); self.expect(&token::CloseDelim(token::Brace))?; return Ok(self.mk_expr(span, ExprKind::Struct(pth, fields, base), attrs)); } @@ -2996,10 +2439,13 @@ impl<'a> Parser<'a> { } /// Parses a block or unsafe block. - fn parse_block_expr(&mut self, opt_label: Option<Label>, - lo: Span, blk_mode: BlockCheckMode, - outer_attrs: ThinVec<Attribute>) - -> PResult<'a, P<Expr>> { + crate fn parse_block_expr( + &mut self, + opt_label: Option<Label>, + lo: Span, + blk_mode: BlockCheckMode, + outer_attrs: ThinVec<Attribute>, + ) -> PResult<'a, P<Expr>> { self.expect(&token::OpenDelim(token::Brace))?; let mut attrs = outer_attrs; @@ -3034,15 +2480,13 @@ impl<'a> Parser<'a> { attrs.extend::<Vec<_>>(expr.attrs.into()); expr.attrs = attrs; match expr.node { - ExprKind::If(..) | ExprKind::IfLet(..) => { - if !expr.attrs.is_empty() { - // Just point to the first attribute in there... - let span = expr.attrs[0].span; - - self.span_err(span, - "attributes are not yet allowed on `if` \ - expressions"); - } + ExprKind::If(..) if !expr.attrs.is_empty() => { + // Just point to the first attribute in there... + let span = expr.attrs[0].span; + + self.span_err(span, + "attributes are not yet allowed on `if` \ + expressions"); } _ => {} } @@ -3051,20 +2495,26 @@ impl<'a> Parser<'a> { ) } - // Assuming we have just parsed `.`, continue parsing into an expression. + fn mk_await_expr(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> { + let span = lo.to(self.prev_span); + let await_expr = self.mk_expr(span, ExprKind::Await(self_arg), ThinVec::new()); + self.recover_from_await_method_call(); + Ok(await_expr) + } + + /// Assuming we have just parsed `.`, continue parsing into an expression. fn parse_dot_suffix(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> { - let segment = self.parse_path_segment(PathStyle::Expr, true)?; + if self.token.span.rust_2018() && self.eat_keyword(kw::Await) { + return self.mk_await_expr(self_arg, lo); + } + + let segment = self.parse_path_segment(PathStyle::Expr)?; self.check_trailing_angle_brackets(&segment, token::OpenDelim(token::Paren)); - Ok(match self.token { + Ok(match self.token.kind { token::OpenDelim(token::Paren) => { // Method call `expr.f()` - let mut args = self.parse_unspanned_seq( - &token::OpenDelim(token::Paren), - &token::CloseDelim(token::Paren), - SeqSep::trailing_allowed(token::Comma), - |p| Ok(p.parse_expr()?) - )?; + let mut args = self.parse_paren_expr_seq()?; args.insert(0, self_arg); let span = lo.to(self.prev_span); @@ -3083,116 +2533,6 @@ impl<'a> Parser<'a> { }) } - /// This function checks if there are trailing angle brackets and produces - /// a diagnostic to suggest removing them. - /// - /// ```ignore (diagnostic) - /// let _ = vec![1, 2, 3].into_iter().collect::<Vec<usize>>>>(); - /// ^^ help: remove extra angle brackets - /// ``` - fn check_trailing_angle_brackets(&mut self, segment: &PathSegment, end: token::Token) { - // This function is intended to be invoked after parsing a path segment where there are two - // cases: - // - // 1. A specific token is expected after the path segment. - // eg. `x.foo(`, `x.foo::<u32>(` (parenthesis - method call), - // `Foo::`, or `Foo::<Bar>::` (mod sep - continued path). - // 2. No specific token is expected after the path segment. - // eg. `x.foo` (field access) - // - // This function is called after parsing `.foo` and before parsing the token `end` (if - // present). This includes any angle bracket arguments, such as `.foo::<u32>` or - // `Foo::<Bar>`. - - // We only care about trailing angle brackets if we previously parsed angle bracket - // arguments. This helps stop us incorrectly suggesting that extra angle brackets be - // removed in this case: - // - // `x.foo >> (3)` (where `x.foo` is a `u32` for example) - // - // This case is particularly tricky as we won't notice it just looking at the tokens - - // it will appear the same (in terms of upcoming tokens) as below (since the `::<u32>` will - // have already been parsed): - // - // `x.foo::<u32>>>(3)` - let parsed_angle_bracket_args = segment.args - .as_ref() - .map(|args| args.is_angle_bracketed()) - .unwrap_or(false); - - debug!( - "check_trailing_angle_brackets: parsed_angle_bracket_args={:?}", - parsed_angle_bracket_args, - ); - if !parsed_angle_bracket_args { - return; - } - - // Keep the span at the start so we can highlight the sequence of `>` characters to be - // removed. - let lo = self.span; - - // We need to look-ahead to see if we have `>` characters without moving the cursor forward - // (since we might have the field access case and the characters we're eating are - // actual operators and not trailing characters - ie `x.foo >> 3`). - let mut position = 0; - - // We can encounter `>` or `>>` tokens in any order, so we need to keep track of how - // many of each (so we can correctly pluralize our error messages) and continue to - // advance. - let mut number_of_shr = 0; - let mut number_of_gt = 0; - while self.look_ahead(position, |t| { - trace!("check_trailing_angle_brackets: t={:?}", t); - if *t == token::BinOp(token::BinOpToken::Shr) { - number_of_shr += 1; - true - } else if *t == token::Gt { - number_of_gt += 1; - true - } else { - false - } - }) { - position += 1; - } - - // If we didn't find any trailing `>` characters, then we have nothing to error about. - debug!( - "check_trailing_angle_brackets: number_of_gt={:?} number_of_shr={:?}", - number_of_gt, number_of_shr, - ); - if number_of_gt < 1 && number_of_shr < 1 { - return; - } - - // Finally, double check that we have our end token as otherwise this is the - // second case. - if self.look_ahead(position, |t| { - trace!("check_trailing_angle_brackets: t={:?}", t); - *t == end - }) { - // Eat from where we started until the end token so that parsing can continue - // as if we didn't have those extra angle brackets. - self.eat_to_tokens(&[&end]); - let span = lo.until(self.span); - - let plural = number_of_gt > 1 || number_of_shr >= 1; - self.diagnostic() - .struct_span_err( - span, - &format!("unmatched angle bracket{}", if plural { "s" } else { "" }), - ) - .span_suggestion( - span, - &format!("remove extra angle bracket{}", if plural { "s" } else { "" }), - String::new(), - Applicability::MachineApplicable, - ) - .emit(); - } - } - fn parse_dot_or_call_expr_with_(&mut self, e0: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> { let mut e = e0; let mut hi; @@ -3205,125 +2545,128 @@ impl<'a> Parser<'a> { // expr.f if self.eat(&token::Dot) { - match self.token { - token::Ident(..) => { - e = self.parse_dot_suffix(e, lo)?; - } - token::Literal(token::Integer(name), _) => { - let span = self.span; - self.bump(); - let field = ExprKind::Field(e, Ident::new(name, span)); - e = self.mk_expr(lo.to(span), field, ThinVec::new()); - } - token::Literal(token::Float(n), _suf) => { - self.bump(); - let fstr = n.as_str(); - let mut err = self.diagnostic() - .struct_span_err(self.prev_span, &format!("unexpected token: `{}`", n)); - err.span_label(self.prev_span, "unexpected token"); - if fstr.chars().all(|x| "0123456789.".contains(x)) { - let float = match fstr.parse::<f64>().ok() { - Some(f) => f, - None => continue, - }; - let sugg = pprust::to_string(|s| { - use crate::print::pprust::PrintState; - s.popen()?; - s.print_expr(&e)?; - s.s.word( ".")?; - s.print_usize(float.trunc() as usize)?; - s.pclose()?; - s.s.word(".")?; - s.s.word(fstr.splitn(2, ".").last().unwrap().to_string()) - }); - err.span_suggestion( - lo.to(self.prev_span), - "try parenthesizing the first index", - sugg, - Applicability::MachineApplicable - ); + match self.token.kind { + token::Ident(..) => { + e = self.parse_dot_suffix(e, lo)?; } - return Err(err); + token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => { + let span = self.token.span; + self.bump(); + let field = ExprKind::Field(e, Ident::new(symbol, span)); + e = self.mk_expr(lo.to(span), field, ThinVec::new()); + + self.expect_no_suffix(span, "a tuple index", suffix); + } + token::Literal(token::Lit { kind: token::Float, symbol, .. }) => { + self.bump(); + let fstr = symbol.as_str(); + let msg = format!("unexpected token: `{}`", symbol); + let mut err = self.diagnostic().struct_span_err(self.prev_span, &msg); + err.span_label(self.prev_span, "unexpected token"); + if fstr.chars().all(|x| "0123456789.".contains(x)) { + let float = match fstr.parse::<f64>().ok() { + Some(f) => f, + None => continue, + }; + let sugg = pprust::to_string(|s| { + s.popen(); + s.print_expr(&e); + s.s.word( "."); + s.print_usize(float.trunc() as usize); + s.pclose(); + s.s.word("."); + s.s.word(fstr.splitn(2, ".").last().unwrap().to_string()) + }); + err.span_suggestion( + lo.to(self.prev_span), + "try parenthesizing the first index", + sugg, + Applicability::MachineApplicable + ); + } + return Err(err); - } - _ => { - // FIXME Could factor this out into non_fatal_unexpected or something. - let actual = self.this_token_to_string(); - self.span_err(self.span, &format!("unexpected token: `{}`", actual)); - } + } + _ => { + // FIXME Could factor this out into non_fatal_unexpected or something. + let actual = self.this_token_to_string(); + self.span_err(self.token.span, &format!("unexpected token: `{}`", actual)); + } } continue; } if self.expr_is_complete(&e) { break; } - match self.token { - // expr(...) - token::OpenDelim(token::Paren) => { - let es = self.parse_unspanned_seq( - &token::OpenDelim(token::Paren), - &token::CloseDelim(token::Paren), - SeqSep::trailing_allowed(token::Comma), - |p| Ok(p.parse_expr()?) - )?; - hi = self.prev_span; - - let nd = self.mk_call(e, es); - e = self.mk_expr(lo.to(hi), nd, ThinVec::new()); - } + match self.token.kind { + // expr(...) + token::OpenDelim(token::Paren) => { + let seq = self.parse_paren_expr_seq().map(|es| { + let nd = self.mk_call(e, es); + let hi = self.prev_span; + self.mk_expr(lo.to(hi), nd, ThinVec::new()) + }); + e = self.recover_seq_parse_error(token::Paren, lo, seq); + } - // expr[...] - // Could be either an index expression or a slicing expression. - token::OpenDelim(token::Bracket) => { - self.bump(); - let ix = self.parse_expr()?; - hi = self.span; - self.expect(&token::CloseDelim(token::Bracket))?; - let index = self.mk_index(e, ix); - e = self.mk_expr(lo.to(hi), index, ThinVec::new()) - } - _ => return Ok(e) + // expr[...] + // Could be either an index expression or a slicing expression. + token::OpenDelim(token::Bracket) => { + self.bump(); + let ix = self.parse_expr()?; + hi = self.token.span; + self.expect(&token::CloseDelim(token::Bracket))?; + let index = self.mk_index(e, ix); + e = self.mk_expr(lo.to(hi), index, ThinVec::new()) + } + _ => return Ok(e) } } return Ok(e); } + fn parse_paren_expr_seq(&mut self) -> PResult<'a, Vec<P<Expr>>> { + self.parse_paren_comma_seq(|p| p.parse_expr()).map(|(r, _)| r) + } + crate fn process_potential_macro_variable(&mut self) { - let (token, span) = match self.token { - token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() && + self.token = match self.token.kind { + token::Dollar if self.token.span.ctxt() != SyntaxContext::empty() && self.look_ahead(1, |t| t.is_ident()) => { self.bump(); - let name = match self.token { - token::Ident(ident, _) => ident, + let name = match self.token.kind { + token::Ident(name, _) => name, _ => unreachable!() }; - let mut err = self.fatal(&format!("unknown macro variable `{}`", name)); - err.span_label(self.span, "unknown macro variable"); - err.emit(); + let span = self.prev_span.to(self.token.span); + self.diagnostic() + .struct_span_fatal(span, &format!("unknown macro variable `{}`", name)) + .span_label(span, "unknown macro variable") + .emit(); self.bump(); return } token::Interpolated(ref nt) => { - self.meta_var_span = Some(self.span); + self.meta_var_span = Some(self.token.span); // Interpolated identifier and lifetime tokens are replaced with usual identifier // and lifetime tokens, so the former are never encountered during normal parsing. match **nt { - token::NtIdent(ident, is_raw) => (token::Ident(ident, is_raw), ident.span), - token::NtLifetime(ident) => (token::Lifetime(ident), ident.span), + token::NtIdent(ident, is_raw) => + Token::new(token::Ident(ident.name, is_raw), ident.span), + token::NtLifetime(ident) => + Token::new(token::Lifetime(ident.name), ident.span), _ => return, } } _ => return, }; - self.token = token; - self.span = span; } /// Parses a single token tree from the input. crate fn parse_token_tree(&mut self) -> TokenTree { - match self.token { + match self.token.kind { token::OpenDelim(..) => { let frame = mem::replace(&mut self.token_cursor.frame, self.token_cursor.stack.pop().unwrap()); - self.span = frame.span.entire(); + self.token.span = frame.span.entire(); self.bump(); TokenTree::Delimited( frame.span, @@ -3333,15 +2676,14 @@ impl<'a> Parser<'a> { }, token::CloseDelim(_) | token::Eof => unreachable!(), _ => { - let (token, span) = (mem::replace(&mut self.token, token::Whitespace), self.span); + let token = self.token.take(); self.bump(); - TokenTree::Token(span, token) + TokenTree::Token(token) } } } - // parse a stream of tokens into a list of TokenTree's, - // up to EOF. + /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF. pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec<TokenTree>> { let mut tts = Vec::new(); while self.token != token::Eof { @@ -3353,7 +2695,7 @@ impl<'a> Parser<'a> { pub fn parse_tokens(&mut self) -> TokenStream { let mut result = Vec::new(); loop { - match self.token { + match self.token.kind { token::Eof | token::CloseDelim(..) => break, _ => result.push(self.parse_token_tree().into()), } @@ -3366,9 +2708,9 @@ impl<'a> Parser<'a> { already_parsed_attrs: Option<ThinVec<Attribute>>) -> PResult<'a, P<Expr>> { let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?; - let lo = self.span; - // Note: when adding new unary operators, don't forget to adjust Token::can_begin_expr() - let (hi, ex) = match self.token { + let lo = self.token.span; + // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr() + let (hi, ex) = match self.token.kind { token::Not => { self.bump(); let e = self.parse_prefix_expr(None); @@ -3381,15 +2723,14 @@ impl<'a> Parser<'a> { let e = self.parse_prefix_expr(None); let (span, e) = self.interpolated_or_expr_span(e)?; let span_of_tilde = lo; - let mut err = self.diagnostic() - .struct_span_err(span_of_tilde, "`~` cannot be used as a unary operator"); - err.span_suggestion_short( - span_of_tilde, - "use `!` to perform bitwise negation", - "!".to_owned(), - Applicability::MachineApplicable - ); - err.emit(); + self.struct_span_err(span_of_tilde, "`~` cannot be used as a unary operator") + .span_suggestion_short( + span_of_tilde, + "use `!` to perform bitwise negation", + "!".to_owned(), + Applicability::MachineApplicable + ) + .emit(); (lo.to(span), self.mk_unary(UnOp::Not, e)) } token::BinOp(token::Minus) => { @@ -3411,58 +2752,41 @@ impl<'a> Parser<'a> { let (span, e) = self.interpolated_or_expr_span(e)?; (lo.to(span), ExprKind::AddrOf(m, e)) } - token::Ident(..) if self.token.is_keyword(keywords::In) => { - self.bump(); - let place = self.parse_expr_res( - Restrictions::NO_STRUCT_LITERAL, - None, - )?; - let blk = self.parse_block()?; - let span = blk.span; - let blk_expr = self.mk_expr(span, ExprKind::Block(blk, None), ThinVec::new()); - (lo.to(span), ExprKind::ObsoleteInPlace(place, blk_expr)) - } - token::Ident(..) if self.token.is_keyword(keywords::Box) => { + token::Ident(..) if self.token.is_keyword(kw::Box) => { self.bump(); let e = self.parse_prefix_expr(None); let (span, e) = self.interpolated_or_expr_span(e)?; (lo.to(span), ExprKind::Box(e)) } - token::Ident(..) if self.token.is_ident_named("not") => { + token::Ident(..) if self.token.is_ident_named(sym::not) => { // `not` is just an ordinary identifier in Rust-the-language, // but as `rustc`-the-compiler, we can issue clever diagnostics // for confused users who really want to say `!` - let token_cannot_continue_expr = |t: &token::Token| match *t { + let token_cannot_continue_expr = |t: &Token| match t.kind { // These tokens can start an expression after `!`, but // can't continue an expression after an ident - token::Ident(ident, is_raw) => token::ident_can_begin_expr(ident, is_raw), + token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw), token::Literal(..) | token::Pound => true, - token::Interpolated(ref nt) => match **nt { - token::NtIdent(..) | token::NtExpr(..) | - token::NtBlock(..) | token::NtPath(..) => true, - _ => false, - }, - _ => false + _ => t.is_whole_expr(), }; let cannot_continue_expr = self.look_ahead(1, token_cannot_continue_expr); if cannot_continue_expr { self.bump(); // Emit the error ... - let mut err = self.diagnostic() - .struct_span_err(self.span, - &format!("unexpected {} after identifier", - self.this_token_descr())); - // span the `not` plus trailing whitespace to avoid - // trailing whitespace after the `!` in our suggestion - let to_replace = self.sess.source_map() - .span_until_non_whitespace(lo.to(self.span)); - err.span_suggestion_short( - to_replace, + self.struct_span_err( + self.token.span, + &format!("unexpected {} after identifier",self.this_token_descr()) + ) + .span_suggestion_short( + // Span the `not` plus trailing whitespace to avoid + // trailing whitespace after the `!` in our suggestion + self.sess.source_map() + .span_until_non_whitespace(lo.to(self.token.span)), "use `!` to perform logical negation", "!".to_owned(), Applicability::MachineApplicable - ); - err.emit(); + ) + .emit(); // —and recover! (just as if we were in the block // for the `token::Not` arm) let e = self.parse_prefix_expr(None); @@ -3489,10 +2813,11 @@ impl<'a> Parser<'a> { } /// Parses an associative expression with operators of at least `min_prec` precedence. - fn parse_assoc_expr_with(&mut self, - min_prec: usize, - lhs: LhsExpr) - -> PResult<'a, P<Expr>> { + fn parse_assoc_expr_with( + &mut self, + min_prec: usize, + lhs: LhsExpr, + ) -> PResult<'a, P<Expr>> { let mut lhs = if let LhsExpr::AlreadyParsed(expr) = lhs { expr } else { @@ -3500,16 +2825,57 @@ impl<'a> Parser<'a> { LhsExpr::AttributesParsed(attrs) => Some(attrs), _ => None, }; - if [token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token) { + if [token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token.kind) { return self.parse_prefix_range_expr(attrs); } else { self.parse_prefix_expr(attrs)? } }; - - if self.expr_is_complete(&lhs) { - // Semi-statement forms are odd. See https://github.com/rust-lang/rust/issues/29071 - return Ok(lhs); + let last_type_ascription_set = self.last_type_ascription.is_some(); + + match (self.expr_is_complete(&lhs), AssocOp::from_token(&self.token)) { + (true, None) => { + self.last_type_ascription = None; + // Semi-statement forms are odd. See https://github.com/rust-lang/rust/issues/29071 + return Ok(lhs); + } + (false, _) => {} // continue parsing the expression + // An exhaustive check is done in the following block, but these are checked first + // because they *are* ambiguous but also reasonable looking incorrect syntax, so we + // want to keep their span info to improve diagnostics in these cases in a later stage. + (true, Some(AssocOp::Multiply)) | // `{ 42 } *foo = bar;` or `{ 42 } * 3` + (true, Some(AssocOp::Subtract)) | // `{ 42 } -5` + (true, Some(AssocOp::LAnd)) | // `{ 42 } &&x` (#61475) + (true, Some(AssocOp::Add)) // `{ 42 } + 42 + // If the next token is a keyword, then the tokens above *are* unambiguously incorrect: + // `if x { a } else { b } && if y { c } else { d }` + if !self.look_ahead(1, |t| t.is_reserved_ident()) => { + self.last_type_ascription = None; + // These cases are ambiguous and can't be identified in the parser alone + let sp = self.sess.source_map().start_point(self.token.span); + self.sess.ambiguous_block_expr_parse.borrow_mut().insert(sp, lhs.span); + return Ok(lhs); + } + (true, Some(ref op)) if !op.can_continue_expr_unambiguously() => { + self.last_type_ascription = None; + return Ok(lhs); + } + (true, Some(_)) => { + // We've found an expression that would be parsed as a statement, but the next + // token implies this should be parsed as an expression. + // For example: `if let Some(x) = x { x } else { 0 } / 2` + let mut err = self.struct_span_err(self.token.span, &format!( + "expected expression, found `{}`", + pprust::token_to_string(&self.token), + )); + err.span_label(self.token.span, "expected expression"); + self.sess.expr_parentheses_needed( + &mut err, + lhs.span, + Some(pprust::expr_to_string(&lhs), + )); + err.emit(); + } } self.expected_tokens.push(TokenType::Operator); while let Some(op) = AssocOp::from_token(&self.token) { @@ -3525,18 +2891,19 @@ impl<'a> Parser<'a> { _ => lhs.span, }; - let cur_op_span = self.span; + let cur_op_span = self.token.span; let restrictions = if op.is_assign_like() { self.restrictions & Restrictions::NO_STRUCT_LITERAL } else { self.restrictions }; - if op.precedence() < min_prec { + let prec = op.precedence(); + if prec < min_prec { break; } // Check for deprecated `...` syntax if self.token == token::DotDotDot && op == AssocOp::DotDotEq { - self.err_dotdotdot_syntax(self.span); + self.err_dotdotdot_syntax(self.token.span); } self.bump(); @@ -3548,25 +2915,10 @@ impl<'a> Parser<'a> { lhs = self.parse_assoc_op_cast(lhs, lhs_span, ExprKind::Cast)?; continue } else if op == AssocOp::Colon { - lhs = match self.parse_assoc_op_cast(lhs, lhs_span, ExprKind::Type) { - Ok(lhs) => lhs, - Err(mut err) => { - err.span_label(self.span, - "expecting a type here because of type ascription"); - let cm = self.sess.source_map(); - let cur_pos = cm.lookup_char_pos(self.span.lo()); - let op_pos = cm.lookup_char_pos(cur_op_span.hi()); - if cur_pos.line != op_pos.line { - err.span_suggestion( - cur_op_span, - "try using a semicolon", - ";".to_string(), - Applicability::MaybeIncorrect // speculative - ); - } - return Err(err); - } - }; + let maybe_path = self.could_ascription_be_path(&lhs.node); + self.last_type_ascription = Some((self.prev_span, maybe_path)); + + lhs = self.parse_assoc_op_cast(lhs, lhs_span, ExprKind::Type)?; continue } else if op == AssocOp::DotDot || op == AssocOp::DotDotEq { // If we didn’t have to handle `x..`/`x..=`, it would be pretty easy to @@ -3575,8 +2927,7 @@ impl<'a> Parser<'a> { // We have 2 alternatives here: `x..y`/`x..=y` and `x..`/`x..=` The other // two variants are handled with `parse_prefix_range_expr` call above. let rhs = if self.is_at_start_of_range_notation_rhs() { - Some(self.parse_assoc_expr_with(op.precedence() + 1, - LhsExpr::NotYetParsed)?) + Some(self.parse_assoc_expr_with(prec + 1, LhsExpr::NotYetParsed)?) } else { None }; @@ -3596,28 +2947,18 @@ impl<'a> Parser<'a> { break } - let rhs = match op.fixity() { - Fixity::Right => self.with_res( - restrictions - Restrictions::STMT_EXPR, - |this| { - this.parse_assoc_expr_with(op.precedence(), - LhsExpr::NotYetParsed) - }), - Fixity::Left => self.with_res( - restrictions - Restrictions::STMT_EXPR, - |this| { - this.parse_assoc_expr_with(op.precedence() + 1, - LhsExpr::NotYetParsed) - }), + let fixity = op.fixity(); + let prec_adjustment = match fixity { + Fixity::Right => 0, + Fixity::Left => 1, // We currently have no non-associative operators that are not handled above by // the special cases. The code is here only for future convenience. - Fixity::None => self.with_res( - restrictions - Restrictions::STMT_EXPR, - |this| { - this.parse_assoc_expr_with(op.precedence() + 1, - LhsExpr::NotYetParsed) - }), - }?; + Fixity::None => 1, + }; + let rhs = self.with_res( + restrictions - Restrictions::STMT_EXPR, + |this| this.parse_assoc_expr_with(prec + prec_adjustment, LhsExpr::NotYetParsed) + )?; // Make sure that the span of the parent node is larger than the span of lhs and rhs, // including the attributes. @@ -3638,10 +2979,7 @@ impl<'a> Parser<'a> { let binary = self.mk_binary(source_map::respan(cur_op_span, ast_op), lhs, rhs); self.mk_expr(span, binary, ThinVec::new()) } - AssocOp::Assign => - self.mk_expr(span, ExprKind::Assign(lhs, rhs), ThinVec::new()), - AssocOp::ObsoleteInPlace => - self.mk_expr(span, ExprKind::ObsoleteInPlace(lhs, rhs), ThinVec::new()), + AssocOp::Assign => self.mk_expr(span, ExprKind::Assign(lhs, rhs), ThinVec::new()), AssocOp::AssignOp(k) => { let aop = match k { token::Plus => BinOpKind::Add, @@ -3663,7 +3001,10 @@ impl<'a> Parser<'a> { } }; - if op.fixity() == Fixity::None { break } + if let Fixity::None = fixity { break } + } + if last_type_ascription_set { + self.last_type_ascription = None; } Ok(lhs) } @@ -3691,7 +3032,7 @@ impl<'a> Parser<'a> { match self.parse_path(PathStyle::Expr) { Ok(path) => { - let (op_noun, op_verb) = match self.token { + let (op_noun, op_verb) = match self.token.kind { token::Lt => ("comparison", "comparing"), token::BinOp(token::Shl) => ("shift", "shifting"), _ => { @@ -3711,26 +3052,29 @@ impl<'a> Parser<'a> { // in AST and continue parsing. let msg = format!("`<` is interpreted as a start of generic \ arguments for `{}`, not a {}", path, op_noun); - let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg); - err.span_label(self.look_ahead_span(1).to(parser_snapshot_after_type.span), - "interpreted as generic arguments"); - err.span_label(self.span, format!("not interpreted as {}", op_noun)); - + let span_after_type = parser_snapshot_after_type.token.span; let expr = mk_expr(self, P(Ty { span: path.span, node: TyKind::Path(None, path), id: ast::DUMMY_NODE_ID })); - let expr_str = self.sess.source_map().span_to_snippet(expr.span) - .unwrap_or_else(|_| pprust::expr_to_string(&expr)); - err.span_suggestion( - expr.span, - &format!("try {} the cast value", op_verb), - format!("({})", expr_str), - Applicability::MachineApplicable - ); - err.emit(); + let expr_str = self.span_to_snippet(expr.span) + .unwrap_or_else(|_| pprust::expr_to_string(&expr)); + + self.struct_span_err(self.token.span, &msg) + .span_label( + self.look_ahead(1, |t| t.span).to(span_after_type), + "interpreted as generic arguments" + ) + .span_label(self.token.span, format!("not interpreted as {}", op_noun)) + .span_suggestion( + expr.span, + &format!("try {} the cast value", op_verb), + format!("({})", expr_str), + Applicability::MachineApplicable + ) + .emit(); Ok(expr) } @@ -3745,49 +3089,22 @@ impl<'a> Parser<'a> { } } - /// Produce an error if comparison operators are chained (RFC #558). - /// We only need to check lhs, not rhs, because all comparison ops - /// have same precedence and are left-associative - fn check_no_chained_comparison(&mut self, lhs: &Expr, outer_op: &AssocOp) { - debug_assert!(outer_op.is_comparison(), - "check_no_chained_comparison: {:?} is not comparison", - outer_op); - match lhs.node { - ExprKind::Binary(op, _, _) if op.node.is_comparison() => { - // respan to include both operators - let op_span = op.span.to(self.span); - let mut err = self.diagnostic().struct_span_err(op_span, - "chained comparison operators require parentheses"); - if op.node == BinOpKind::Lt && - *outer_op == AssocOp::Less || // Include `<` to provide this recommendation - *outer_op == AssocOp::Greater // even in a case like the following: - { // Foo<Bar<Baz<Qux, ()>>> - err.help( - "use `::<...>` instead of `<...>` if you meant to specify type arguments"); - err.help("or use `(...)` if you meant to specify fn arguments"); - } - err.emit(); - } - _ => {} - } - } - /// Parse prefix-forms of range notation: `..expr`, `..`, `..=expr` fn parse_prefix_range_expr(&mut self, already_parsed_attrs: Option<ThinVec<Attribute>>) -> PResult<'a, P<Expr>> { // Check for deprecated `...` syntax if self.token == token::DotDotDot { - self.err_dotdotdot_syntax(self.span); + self.err_dotdotdot_syntax(self.token.span); } - debug_assert!([token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token), + debug_assert!([token::DotDot, token::DotDotDot, token::DotDotEq].contains(&self.token.kind), "parse_prefix_range_expr: token {:?} is not DotDot/DotDotEq", self.token); let tok = self.token.clone(); let attrs = self.parse_or_use_outer_attributes(already_parsed_attrs)?; - let lo = self.span; - let mut hi = self.span; + let lo = self.token.span; + let mut hi = self.token.span; self.bump(); let opt_end = if self.is_at_start_of_range_notation_rhs() { // RHS must be parsed with more associativity than the dots. @@ -3798,7 +3115,7 @@ impl<'a> Parser<'a> { hi = x.span; x })?) - } else { + } else { None }; let limits = if tok == token::DotDot { @@ -3823,19 +3140,16 @@ impl<'a> Parser<'a> { } } - /// Parses an `if` or `if let` expression (`if` token already eaten). + /// Parses an `if` expression (`if` token already eaten). fn parse_if_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - if self.check_keyword(keywords::Let) { - return self.parse_if_let_expr(attrs); - } let lo = self.prev_span; - let cond = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; + let cond = self.parse_cond_expr()?; // Verify that the parsed `if` condition makes sense as a condition. If it is a block, then // verify that the last statement is either an implicit return (no `;`) or an explicit // return. This won't catch blocks with an explicit `return`, but that would be caught by // the dead code lint. - if self.eat_keyword(keywords::Else) || !cond.returns() { + if self.eat_keyword(kw::Else) || !cond.returns() { let sp = self.sess.source_map().next_point(lo); let mut err = self.diagnostic() .struct_span_err(sp, "missing condition for `if` statemement"); @@ -3851,7 +3165,7 @@ impl<'a> Parser<'a> { })?; let mut els: Option<P<Expr>> = None; let mut hi = thn.span; - if self.eat_keyword(keywords::Else) { + if self.eat_keyword(kw::Else) { let elexpr = self.parse_else_expr()?; hi = elexpr.span; els = Some(elexpr); @@ -3859,22 +3173,32 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(lo.to(hi), ExprKind::If(cond, thn, els), attrs)) } - /// Parses an `if let` expression (`if` token already eaten). - fn parse_if_let_expr(&mut self, attrs: ThinVec<Attribute>) - -> PResult<'a, P<Expr>> { + /// Parse the condition of a `if`- or `while`-expression + fn parse_cond_expr(&mut self) -> PResult<'a, P<Expr>> { + let cond = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; + + if let ExprKind::Let(..) = cond.node { + // Remove the last feature gating of a `let` expression since it's stable. + let last = self.sess.let_chains_spans.borrow_mut().pop(); + debug_assert_eq!(cond.span, last.unwrap()); + } + + Ok(cond) + } + + /// Parses a `let $pats = $expr` pseudo-expression. + /// The `let` token has already been eaten. + fn parse_let_expr(&mut self, attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { let lo = self.prev_span; - self.expect_keyword(keywords::Let)?; let pats = self.parse_pats()?; self.expect(&token::Eq)?; - let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; - let thn = self.parse_block()?; - let (hi, els) = if self.eat_keyword(keywords::Else) { - let expr = self.parse_else_expr()?; - (expr.span, Some(expr)) - } else { - (thn.span, None) - }; - Ok(self.mk_expr(lo.to(hi), ExprKind::IfLet(pats, expr, thn, els), attrs)) + let expr = self.with_res( + Restrictions::NO_STRUCT_LITERAL, + |this| this.parse_assoc_expr_with(1 + prec_let_scrutinee_needs_par(), None.into()) + )?; + let span = lo.to(expr.span); + self.sess.let_chains_spans.borrow_mut().push(span); + Ok(self.mk_expr(span, ExprKind::Let(pats, expr), attrs)) } /// Parses `move |args| expr`. @@ -3882,22 +3206,25 @@ impl<'a> Parser<'a> { attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - let lo = self.span; - let movability = if self.eat_keyword(keywords::Static) { + let lo = self.token.span; + + let movability = if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable }; - let asyncness = if self.span.rust_2018() { + + let asyncness = if self.token.span.rust_2018() { self.parse_asyncness() } else { IsAsync::NotAsync }; - let capture_clause = if self.eat_keyword(keywords::Move) { - CaptureBy::Value - } else { - CaptureBy::Ref - }; + if asyncness.is_async() { + // Feature gate `async ||` closures. + self.sess.async_closure_spans.borrow_mut().push(self.prev_span); + } + + let capture_clause = self.parse_capture_clause(); let decl = self.parse_fn_block_decl()?; let decl_hi = self.prev_span; let body = match decl.output { @@ -3908,7 +3235,7 @@ impl<'a> Parser<'a> { _ => { // If an explicit return type is given, require a // block to appear (RFC 968). - let body_lo = self.span; + let body_lo = self.token.span; self.parse_block_expr(None, body_lo, BlockCheckMode::Default, ThinVec::new())? } }; @@ -3919,9 +3246,9 @@ impl<'a> Parser<'a> { attrs)) } - // `else` token already eaten + /// `else` token already eaten fn parse_else_expr(&mut self) -> PResult<'a, P<Expr>> { - if self.eat_keyword(keywords::If) { + if self.eat_keyword(kw::If) { return self.parse_if_expr(ThinVec::new()); } else { let blk = self.parse_block()?; @@ -3930,42 +3257,40 @@ impl<'a> Parser<'a> { } /// Parse a 'for' .. 'in' expression ('for' token already eaten) - fn parse_for_expr(&mut self, opt_label: Option<Label>, - span_lo: Span, - mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { + fn parse_for_expr( + &mut self, + opt_label: Option<Label>, + span_lo: Span, + mut attrs: ThinVec<Attribute> + ) -> PResult<'a, P<Expr>> { // Parse: `for <src_pat> in <src_expr> <src_loop_block>` + // Record whether we are about to parse `for (`. + // This is used below for recovery in case of `for ( $stuff ) $block` + // in which case we will suggest `for $stuff $block`. + let begin_paren = match self.token.kind { + token::OpenDelim(token::Paren) => Some(self.token.span), + _ => None, + }; + let pat = self.parse_top_level_pat()?; - if !self.eat_keyword(keywords::In) { - let in_span = self.prev_span.between(self.span); - let mut err = self.sess.span_diagnostic - .struct_span_err(in_span, "missing `in` in `for` loop"); - err.span_suggestion_short( - in_span, "try adding `in` here", " in ".into(), - // has been misleading, at least in the past (closed Issue #48492) - Applicability::MaybeIncorrect - ); - err.emit(); + if !self.eat_keyword(kw::In) { + let in_span = self.prev_span.between(self.token.span); + self.struct_span_err(in_span, "missing `in` in `for` loop") + .span_suggestion_short( + in_span, + "try adding `in` here", " in ".into(), + // has been misleading, at least in the past (closed Issue #48492) + Applicability::MaybeIncorrect + ) + .emit(); } let in_span = self.prev_span; - if self.eat_keyword(keywords::In) { - // a common typo: `for _ in in bar {}` - let mut err = self.sess.span_diagnostic.struct_span_err( - self.prev_span, - "expected iterable, found keyword `in`", - ); - err.span_suggestion_short( - in_span.until(self.prev_span), - "remove the duplicated `in`", - String::new(), - Applicability::MachineApplicable, - ); - err.note("if you meant to use emplacement syntax, it is obsolete (for now, anyway)"); - err.note("for more information on the status of emplacement syntax, see <\ - https://github.com/rust-lang/rust/issues/27779#issuecomment-378416911>"); - err.emit(); - } + self.check_for_for_in_in_typo(in_span); let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; + + let pat = self.recover_parens_around_for_head(pat, &expr, begin_paren); + let (iattrs, loop_block) = self.parse_inner_attrs_and_block()?; attrs.extend(iattrs); @@ -3977,31 +3302,14 @@ impl<'a> Parser<'a> { fn parse_while_expr(&mut self, opt_label: Option<Label>, span_lo: Span, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - if self.token.is_keyword(keywords::Let) { - return self.parse_while_let_expr(opt_label, span_lo, attrs); - } - let cond = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; + let cond = self.parse_cond_expr()?; let (iattrs, body) = self.parse_inner_attrs_and_block()?; attrs.extend(iattrs); let span = span_lo.to(body.span); - return Ok(self.mk_expr(span, ExprKind::While(cond, body, opt_label), attrs)); + Ok(self.mk_expr(span, ExprKind::While(cond, body, opt_label), attrs)) } - /// Parses a `while let` expression (`while` token already eaten). - fn parse_while_let_expr(&mut self, opt_label: Option<Label>, - span_lo: Span, - mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { - self.expect_keyword(keywords::Let)?; - let pats = self.parse_pats()?; - self.expect(&token::Eq)?; - let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; - let (iattrs, body) = self.parse_inner_attrs_and_block()?; - attrs.extend(iattrs); - let span = span_lo.to(body.span); - return Ok(self.mk_expr(span, ExprKind::WhileLet(pats, expr, body, opt_label), attrs)); - } - - // parse `loop {...}`, `loop` token already eaten + /// Parse `loop {...}`, `loop` token already eaten. fn parse_loop_expr(&mut self, opt_label: Option<Label>, span_lo: Span, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { @@ -4011,17 +3319,20 @@ impl<'a> Parser<'a> { Ok(self.mk_expr(span, ExprKind::Loop(body, opt_label), attrs)) } - /// Parses an `async move {...}` expression. - pub fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) - -> PResult<'a, P<Expr>> - { - let span_lo = self.span; - self.expect_keyword(keywords::Async)?; - let capture_clause = if self.eat_keyword(keywords::Move) { + /// Parse an optional `move` prefix to a closure lke construct. + fn parse_capture_clause(&mut self) -> CaptureBy { + if self.eat_keyword(kw::Move) { CaptureBy::Value } else { CaptureBy::Ref - }; + } + } + + /// Parses an `async move? {...}` expression. + pub fn parse_async_block(&mut self, mut attrs: ThinVec<Attribute>) -> PResult<'a, P<Expr>> { + let span_lo = self.token.span; + self.expect_keyword(kw::Async)?; + let capture_clause = self.parse_capture_clause(); let (iattrs, body) = self.parse_inner_attrs_and_block()?; attrs.extend(iattrs); Ok(self.mk_expr( @@ -4035,7 +3346,15 @@ impl<'a> Parser<'a> { { let (iattrs, body) = self.parse_inner_attrs_and_block()?; attrs.extend(iattrs); - Ok(self.mk_expr(span_lo.to(body.span), ExprKind::TryBlock(body), attrs)) + if self.eat_keyword(kw::Catch) { + let mut error = self.struct_span_err(self.prev_span, + "keyword `catch` cannot follow a `try` block"); + error.help("try using `match` on the result of the `try` block instead"); + error.emit(); + Err(error) + } else { + Ok(self.mk_expr(span_lo.to(body.span), ExprKind::TryBlock(body), attrs)) + } } // `match` token already eaten @@ -4045,7 +3364,7 @@ impl<'a> Parser<'a> { let discriminant = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?; if let Err(mut e) = self.expect(&token::OpenDelim(token::Brace)) { - if self.token == token::Token::Semi { + if self.token == token::Semi { e.span_suggestion_short( match_span, "try removing this `match`", @@ -4065,7 +3384,7 @@ impl<'a> Parser<'a> { // Recover by skipping to the end of the block. e.emit(); self.recover_stmt(); - let span = lo.to(self.span); + let span = lo.to(self.token.span); if self.token == token::CloseDelim(token::Brace) { self.bump(); } @@ -4073,24 +3392,23 @@ impl<'a> Parser<'a> { } } } - let hi = self.span; + let hi = self.token.span; self.bump(); return Ok(self.mk_expr(lo.to(hi), ExprKind::Match(discriminant, arms), attrs)); } crate fn parse_arm(&mut self) -> PResult<'a, Arm> { - maybe_whole!(self, NtArm, |x| x); - let attrs = self.parse_outer_attributes()?; + let lo = self.token.span; let pats = self.parse_pats()?; - let guard = if self.eat_keyword(keywords::If) { - Some(Guard::If(self.parse_expr()?)) + let guard = if self.eat_keyword(kw::If) { + Some(self.parse_expr()?) } else { None }; - let arrow_span = self.span; + let arrow_span = self.token.span; self.expect(&token::FatArrow)?; - let arm_start_span = self.span; + let arm_start_span = self.token.span; let expr = self.parse_expr_res(Restrictions::STMT_EXPR, None) .map_err(|mut err| { @@ -4101,6 +3419,8 @@ impl<'a> Parser<'a> { let require_comma = classify::expr_requires_semi_to_be_stmt(&expr) && self.token != token::CloseDelim(token::Brace); + let hi = self.token.span; + if require_comma { let cm = self.sess.source_map(); self.expect_one_of(&[token::Comma], &[token::CloseDelim(token::Brace)]) @@ -4118,7 +3438,7 @@ impl<'a> Parser<'a> { // | | // | arrow_span // X | &X => "x" - // | - ^^ self.span + // | - ^^ self.token.span // | | // | parsed until here as `"y" & X` err.span_suggestion_short( @@ -4144,6 +3464,7 @@ impl<'a> Parser<'a> { pats, guard, body: expr, + span: lo.to(hi), }) } @@ -4196,15 +3517,14 @@ impl<'a> Parser<'a> { pats.push(self.parse_top_level_pat()?); if self.token == token::OrOr { - let mut err = self.struct_span_err(self.span, - "unexpected token `||` after pattern"); - err.span_suggestion( - self.span, - "use a single `|` to specify multiple patterns", - "|".to_owned(), - Applicability::MachineApplicable - ); - err.emit(); + self.struct_span_err(self.token.span, "unexpected token `||` after pattern") + .span_suggestion( + self.token.span, + "use a single `|` to specify multiple patterns", + "|".to_owned(), + Applicability::MachineApplicable + ) + .emit(); self.bump(); } else if self.eat(&token::BinOp(token::Or)) { // This is a No-op. Continue the loop to parse the next @@ -4215,110 +3535,6 @@ impl<'a> Parser<'a> { }; } - // Parses a parenthesized list of patterns like - // `()`, `(p)`, `(p,)`, `(p, q)`, or `(p, .., q)`. Returns: - // - a vector of the patterns that were parsed - // - an option indicating the index of the `..` element - // - a boolean indicating whether a trailing comma was present. - // Trailing commas are significant because (p) and (p,) are different patterns. - fn parse_parenthesized_pat_list(&mut self) -> PResult<'a, (Vec<P<Pat>>, Option<usize>, bool)> { - self.expect(&token::OpenDelim(token::Paren))?; - let result = self.parse_pat_list()?; - self.expect(&token::CloseDelim(token::Paren))?; - Ok(result) - } - - fn parse_pat_list(&mut self) -> PResult<'a, (Vec<P<Pat>>, Option<usize>, bool)> { - let mut fields = Vec::new(); - let mut ddpos = None; - let mut trailing_comma = false; - loop { - if self.eat(&token::DotDot) { - if ddpos.is_none() { - ddpos = Some(fields.len()); - } else { - // Emit a friendly error, ignore `..` and continue parsing - self.struct_span_err( - self.prev_span, - "`..` can only be used once per tuple or tuple struct pattern", - ) - .span_label(self.prev_span, "can only be used once per pattern") - .emit(); - } - } else if !self.check(&token::CloseDelim(token::Paren)) { - fields.push(self.parse_pat(None)?); - } else { - break - } - - trailing_comma = self.eat(&token::Comma); - if !trailing_comma { - break - } - } - - if ddpos == Some(fields.len()) && trailing_comma { - // `..` needs to be followed by `)` or `, pat`, `..,)` is disallowed. - let msg = "trailing comma is not permitted after `..`"; - self.struct_span_err(self.prev_span, msg) - .span_label(self.prev_span, msg) - .emit(); - } - - Ok((fields, ddpos, trailing_comma)) - } - - fn parse_pat_vec_elements( - &mut self, - ) -> PResult<'a, (Vec<P<Pat>>, Option<P<Pat>>, Vec<P<Pat>>)> { - let mut before = Vec::new(); - let mut slice = None; - let mut after = Vec::new(); - let mut first = true; - let mut before_slice = true; - - while self.token != token::CloseDelim(token::Bracket) { - if first { - first = false; - } else { - self.expect(&token::Comma)?; - - if self.token == token::CloseDelim(token::Bracket) - && (before_slice || !after.is_empty()) { - break - } - } - - if before_slice { - if self.eat(&token::DotDot) { - - if self.check(&token::Comma) || - self.check(&token::CloseDelim(token::Bracket)) { - slice = Some(P(Pat { - id: ast::DUMMY_NODE_ID, - node: PatKind::Wild, - span: self.prev_span, - })); - before_slice = false; - } - continue - } - } - - let subpat = self.parse_pat(None)?; - if before_slice && self.eat(&token::DotDot) { - slice = Some(subpat); - before_slice = false; - } else if before_slice { - before.push(subpat); - } else { - after.push(subpat); - } - } - - Ok((before, slice, after)) - } - fn parse_pat_field( &mut self, lo: Span, @@ -4335,10 +3551,10 @@ impl<'a> Parser<'a> { (pat, fieldname, false) } else { // Parsing a pattern of the form "(box) (ref) (mut) fieldname" - let is_box = self.eat_keyword(keywords::Box); - let boxed_span = self.span; - let is_ref = self.eat_keyword(keywords::Ref); - let is_mut = self.eat_keyword(keywords::Mut); + let is_box = self.eat_keyword(kw::Box); + let boxed_span = self.token.span; + let is_ref = self.eat_keyword(kw::Ref); + let is_mut = self.eat_keyword(kw::Mut); let fieldname = self.parse_ident()?; hi = self.prev_span; @@ -4386,8 +3602,16 @@ impl<'a> Parser<'a> { let mut etc_span = None; while self.token != token::CloseDelim(token::Brace) { - let attrs = self.parse_outer_attributes()?; - let lo = self.span; + let attrs = match self.parse_outer_attributes() { + Ok(attrs) => attrs, + Err(err) => { + if let Some(mut delayed) = delayed_err { + delayed.emit(); + } + return Err(err); + }, + }; + let lo = self.token.span; // check that a comma comes after every field if !ate_comma { @@ -4401,19 +3625,18 @@ impl<'a> Parser<'a> { if self.check(&token::DotDot) || self.token == token::DotDotDot { etc = true; - let mut etc_sp = self.span; + let mut etc_sp = self.token.span; if self.token == token::DotDotDot { // Issue #46718 // Accept `...` as if it were `..` to avoid further errors - let mut err = self.struct_span_err(self.span, - "expected field pattern, found `...`"); - err.span_suggestion( - self.span, - "to omit remaining fields, use one fewer `.`", - "..".to_owned(), - Applicability::MachineApplicable - ); - err.emit(); + self.struct_span_err(self.token.span, "expected field pattern, found `...`") + .span_suggestion( + self.token.span, + "to omit remaining fields, use one fewer `.`", + "..".to_owned(), + Applicability::MachineApplicable + ) + .emit(); } self.bump(); // `..` || `...` @@ -4424,18 +3647,19 @@ impl<'a> Parser<'a> { let token_str = self.this_token_descr(); let mut err = self.fatal(&format!("expected `}}`, found {}", token_str)); - err.span_label(self.span, "expected `}`"); + err.span_label(self.token.span, "expected `}`"); let mut comma_sp = None; if self.token == token::Comma { // Issue #49257 - etc_sp = etc_sp.to(self.sess.source_map().span_until_non_whitespace(self.span)); + let nw_span = self.sess.source_map().span_until_non_whitespace(self.token.span); + etc_sp = etc_sp.to(nw_span); err.span_label(etc_sp, "`..` must be at the end and cannot have a trailing comma"); - comma_sp = Some(self.span); + comma_sp = Some(self.token.span); self.bump(); ate_comma = true; } - etc_span = Some(etc_sp.until(self.span)); + etc_span = Some(etc_sp.until(self.token.span)); if self.token == token::CloseDelim(token::Brace) { // If the struct looks otherwise well formed, recover and continue. if let Some(sp) = comma_sp { @@ -4485,7 +3709,7 @@ impl<'a> Parser<'a> { "move the `..` to the end of the field list", vec![ (etc_span, String::new()), - (self.span, format!("{}.. }}", if ate_comma { "" } else { ", " })), + (self.token.span, format!("{}.. }}", if ate_comma { "" } else { ", " })), ], Applicability::MachineApplicable, ); @@ -4497,7 +3721,7 @@ impl<'a> Parser<'a> { fn parse_pat_range_end(&mut self) -> PResult<'a, P<Expr>> { if self.token.is_path_start() { - let lo = self.span; + let lo = self.token.span; let (qself, path) = if self.eat_lt() { // Parse a qualified path let (qself, path) = self.parse_qpath(PathStyle::Expr)?; @@ -4513,20 +3737,35 @@ impl<'a> Parser<'a> { } } - // helper function to decide whether to parse as ident binding or to try to do - // something more complex like range patterns + /// Is the current token suitable as the start of a range patterns end? + fn is_pat_range_end_start(&self) -> bool { + self.token.is_path_start() // e.g. `MY_CONST`; + || self.token == token::Dot // e.g. `.5` for recovery; + || self.token.can_begin_literal_or_bool() // e.g. `42`. + || self.token.is_whole_expr() + } + + // Helper function to decide whether to parse as ident binding + // or to try to do something more complex like range patterns. fn parse_as_ident(&mut self) -> bool { - self.look_ahead(1, |t| match *t { + self.look_ahead(1, |t| match t.kind { token::OpenDelim(token::Paren) | token::OpenDelim(token::Brace) | - token::DotDotDot | token::DotDotEq | token::ModSep | token::Not => Some(false), - // ensure slice patterns [a, b.., c] and [a, b, c..] don't go into the - // range pattern branch - token::DotDot => None, - _ => Some(true), - }).unwrap_or_else(|| self.look_ahead(2, |t| match *t { - token::Comma | token::CloseDelim(token::Bracket) => true, - _ => false, - })) + token::DotDotDot | token::DotDotEq | token::DotDot | + token::ModSep | token::Not => false, + _ => true, + }) + } + + /// Parse and throw away a parentesized comma separated + /// sequence of patterns until `)` is reached. + fn skip_pat_list(&mut self) -> PResult<'a, ()> { + while !self.check(&token::CloseDelim(token::Paren)) { + self.parse_pat(None)?; + if !self.eat(&token::Comma) { + return Ok(()) + } + } + Ok(()) } /// A wrapper around `parse_pat` with some special error handling for the @@ -4540,9 +3779,9 @@ impl<'a> Parser<'a> { // parentheses in what should have been a tuple pattern; return a // suggestion-enhanced error here rather than choking on the comma // later. - let comma_span = self.span; + let comma_span = self.token.span; self.bump(); - if let Err(mut err) = self.parse_pat_list() { + if let Err(mut err) = self.skip_pat_list() { // We didn't expect this to work anyway; we just wanted // to advance to the end of the comma-sequence so we know // the span to suggest parenthesizing @@ -4551,7 +3790,7 @@ impl<'a> Parser<'a> { let seq_span = pat.span.to(self.prev_span); let mut err = self.struct_span_err(comma_span, "unexpected `,` in pattern"); - if let Ok(seq_snippet) = self.sess.source_map().span_to_snippet(seq_span) { + if let Ok(seq_snippet) = self.span_to_snippet(seq_span) { err.span_suggestion( seq_span, "try adding parentheses to match on a tuple..", @@ -4574,6 +3813,53 @@ impl<'a> Parser<'a> { self.parse_pat_with_range_pat(true, expected) } + /// Parse a range-to pattern, e.g. `..X` and `..=X` for recovery. + fn parse_pat_range_to(&mut self, re: RangeEnd, form: &str) -> PResult<'a, PatKind> { + let lo = self.prev_span; + let end = self.parse_pat_range_end()?; + let range_span = lo.to(end.span); + let begin = self.mk_expr(range_span, ExprKind::Err, ThinVec::new()); + + self.diagnostic() + .struct_span_err(range_span, &format!("`{}X` range patterns are not supported", form)) + .span_suggestion( + range_span, + "try using the minimum value for the type", + format!("MIN{}{}", form, pprust::expr_to_string(&end)), + Applicability::HasPlaceholders, + ) + .emit(); + + Ok(PatKind::Range(begin, end, respan(lo, re))) + } + + /// Parse the end of a `X..Y`, `X..=Y`, or `X...Y` range pattern or recover + /// if that end is missing treating it as `X..`, `X..=`, or `X...` respectively. + fn parse_pat_range_end_opt(&mut self, begin: &Expr, form: &str) -> PResult<'a, P<Expr>> { + if self.is_pat_range_end_start() { + // Parsing e.g. `X..=Y`. + self.parse_pat_range_end() + } else { + // Parsing e.g. `X..`. + let range_span = begin.span.to(self.prev_span); + + self.diagnostic() + .struct_span_err( + range_span, + &format!("`X{}` range patterns are not supported", form), + ) + .span_suggestion( + range_span, + "try using the maximum value for the type", + format!("{}{}MAX", pprust::expr_to_string(&begin), form), + Applicability::HasPlaceholders, + ) + .emit(); + + Ok(self.mk_expr(range_span, ExprKind::Err, ThinVec::new())) + } + } + /// Parses a pattern, with a setting whether modern range patterns (e.g., `a..=b`, `a..b` are /// allowed). fn parse_pat_with_range_pat( @@ -4581,48 +3867,69 @@ impl<'a> Parser<'a> { allow_range_pat: bool, expected: Option<&'static str>, ) -> PResult<'a, P<Pat>> { + maybe_recover_from_interpolated_ty_qpath!(self, true); maybe_whole!(self, NtPat, |x| x); - let lo = self.span; + let lo = self.token.span; let pat; - match self.token { + match self.token.kind { token::BinOp(token::And) | token::AndAnd => { // Parse &pat / &mut pat self.expect_and()?; let mutbl = self.parse_mutability(); - if let token::Lifetime(ident) = self.token { - let mut err = self.fatal(&format!("unexpected lifetime `{}` in pattern", - ident)); - err.span_label(self.span, "unexpected lifetime"); + if let token::Lifetime(name) = self.token.kind { + let mut err = self.fatal(&format!("unexpected lifetime `{}` in pattern", name)); + err.span_label(self.token.span, "unexpected lifetime"); return Err(err); } let subpat = self.parse_pat_with_range_pat(false, expected)?; pat = PatKind::Ref(subpat, mutbl); } token::OpenDelim(token::Paren) => { - // Parse (pat,pat,pat,...) as tuple pattern - let (fields, ddpos, trailing_comma) = self.parse_parenthesized_pat_list()?; - pat = if fields.len() == 1 && ddpos.is_none() && !trailing_comma { + // Parse a tuple or parenthesis pattern. + let (fields, trailing_comma) = self.parse_paren_comma_seq(|p| p.parse_pat(None))?; + + // Here, `(pat,)` is a tuple pattern. + // For backward compatibility, `(..)` is a tuple pattern as well. + pat = if fields.len() == 1 && !(trailing_comma || fields[0].is_rest()) { PatKind::Paren(fields.into_iter().nth(0).unwrap()) } else { - PatKind::Tuple(fields, ddpos) + PatKind::Tuple(fields) }; } token::OpenDelim(token::Bracket) => { - // Parse [pat,pat,...] as slice pattern + // Parse `[pat, pat,...]` as a slice pattern. + let (slice, _) = self.parse_delim_comma_seq(token::Bracket, |p| p.parse_pat(None))?; + pat = PatKind::Slice(slice); + } + token::DotDot => { + self.bump(); + pat = if self.is_pat_range_end_start() { + // Parse `..42` for recovery. + self.parse_pat_range_to(RangeEnd::Excluded, "..")? + } else { + // A rest pattern `..`. + PatKind::Rest + }; + } + token::DotDotEq => { + // Parse `..=42` for recovery. + self.bump(); + pat = self.parse_pat_range_to(RangeEnd::Included(RangeSyntax::DotDotEq), "..=")?; + } + token::DotDotDot => { + // Parse `...42` for recovery. self.bump(); - let (before, slice, after) = self.parse_pat_vec_elements()?; - self.expect(&token::CloseDelim(token::Bracket))?; - pat = PatKind::Slice(before, slice, after); + pat = self.parse_pat_range_to(RangeEnd::Included(RangeSyntax::DotDotDot), "...")?; } // At this point, token != &, &&, (, [ - _ => if self.eat_keyword(keywords::Underscore) { + _ => if self.eat_keyword(kw::Underscore) { // Parse _ pat = PatKind::Wild; - } else if self.eat_keyword(keywords::Mut) { + } else if self.eat_keyword(kw::Mut) { // Parse mut ident @ pat / mut ref ident @ pat - let mutref_span = self.prev_span.to(self.span); - let binding_mode = if self.eat_keyword(keywords::Ref) { + let mutref_span = self.prev_span.to(self.token.span); + let binding_mode = if self.eat_keyword(kw::Ref) { self.diagnostic() .struct_span_err(mutref_span, "the order of `mut` and `ref` is incorrect") .span_suggestion( @@ -4636,11 +3943,11 @@ impl<'a> Parser<'a> { BindingMode::ByValue(Mutability::Mutable) }; pat = self.parse_pat_ident(binding_mode)?; - } else if self.eat_keyword(keywords::Ref) { + } else if self.eat_keyword(kw::Ref) { // Parse ref ident @ pat / ref mut ident @ pat let mutbl = self.parse_mutability(); pat = self.parse_pat_ident(BindingMode::ByRef(mutbl))?; - } else if self.eat_keyword(keywords::Box) { + } else if self.eat_keyword(kw::Box) { // Parse box pat let subpat = self.parse_pat_with_range_pat(false, None)?; pat = PatKind::Box(subpat); @@ -4661,36 +3968,40 @@ impl<'a> Parser<'a> { // Parse an unqualified path (None, self.parse_path(PathStyle::Expr)?) }; - match self.token { + match self.token.kind { token::Not if qself.is_none() => { // Parse macro invocation self.bump(); let (delim, tts) = self.expect_delimited_token_tree()?; - let mac = respan(lo.to(self.prev_span), Mac_ { path, tts, delim }); + let mac = respan(lo.to(self.prev_span), Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + }); pat = PatKind::Mac(mac); } token::DotDotDot | token::DotDotEq | token::DotDot => { - let end_kind = match self.token { - token::DotDot => RangeEnd::Excluded, - token::DotDotDot => RangeEnd::Included(RangeSyntax::DotDotDot), - token::DotDotEq => RangeEnd::Included(RangeSyntax::DotDotEq), + let (end_kind, form) = match self.token.kind { + token::DotDot => (RangeEnd::Excluded, ".."), + token::DotDotDot => (RangeEnd::Included(RangeSyntax::DotDotDot), "..."), + token::DotDotEq => (RangeEnd::Included(RangeSyntax::DotDotEq), "..="), _ => panic!("can only parse `..`/`...`/`..=` for ranges \ (checked above)"), }; - let op_span = self.span; + let op_span = self.token.span; // Parse range let span = lo.to(self.prev_span); let begin = self.mk_expr(span, ExprKind::Path(qself, path), ThinVec::new()); self.bump(); - let end = self.parse_pat_range_end()?; - let op = Spanned { span: op_span, node: end_kind }; - pat = PatKind::Range(begin, end, op); + let end = self.parse_pat_range_end_opt(&begin, form)?; + pat = PatKind::Range(begin, end, respan(op_span, end_kind)); } token::OpenDelim(token::Brace) => { if qself.is_some() { let msg = "unexpected `{` after qualified path"; let mut err = self.fatal(msg); - err.span_label(self.span, msg); + err.span_label(self.token.span, msg); return Err(err); } // Parse struct pattern @@ -4698,7 +4009,7 @@ impl<'a> Parser<'a> { let (fields, etc) = self.parse_pat_fields().unwrap_or_else(|mut e| { e.emit(); self.recover_stmt(); - (vec![], false) + (vec![], true) }); self.bump(); pat = PatKind::Struct(path, fields, etc); @@ -4707,12 +4018,12 @@ impl<'a> Parser<'a> { if qself.is_some() { let msg = "unexpected `(` after qualified path"; let mut err = self.fatal(msg); - err.span_label(self.span, msg); + err.span_label(self.token.span, msg); return Err(err); } // Parse tuple struct or enum pattern - let (fields, ddpos, _) = self.parse_parenthesized_pat_list()?; - pat = PatKind::TupleStruct(path, fields, ddpos) + let (fields, _) = self.parse_paren_comma_seq(|p| p.parse_pat(None))?; + pat = PatKind::TupleStruct(path, fields) } _ => pat = PatKind::Path(qself, path), } @@ -4720,22 +4031,21 @@ impl<'a> Parser<'a> { // Try to parse everything else as literal with optional minus match self.parse_literal_maybe_minus() { Ok(begin) => { - let op_span = self.span; + let op_span = self.token.span; if self.check(&token::DotDot) || self.check(&token::DotDotEq) || self.check(&token::DotDotDot) { - let end_kind = if self.eat(&token::DotDotDot) { - RangeEnd::Included(RangeSyntax::DotDotDot) + let (end_kind, form) = if self.eat(&token::DotDotDot) { + (RangeEnd::Included(RangeSyntax::DotDotDot), "...") } else if self.eat(&token::DotDotEq) { - RangeEnd::Included(RangeSyntax::DotDotEq) + (RangeEnd::Included(RangeSyntax::DotDotEq), "..=") } else if self.eat(&token::DotDot) { - RangeEnd::Excluded + (RangeEnd::Excluded, "..") } else { panic!("impossible case: we already matched \ on a range-operator token") }; - let end = self.parse_pat_range_end()?; - let op = Spanned { span: op_span, node: end_kind }; - pat = PatKind::Range(begin, end, op); + let end = self.parse_pat_range_end_opt(&begin, form)?; + pat = PatKind::Range(begin, end, respan(op_span, end_kind)) } else { pat = PatKind::Lit(begin); } @@ -4749,14 +4059,18 @@ impl<'a> Parser<'a> { self.this_token_descr(), ); let mut err = self.fatal(&msg); - err.span_label(self.span, format!("expected {}", expected)); + err.span_label(self.token.span, format!("expected {}", expected)); + let sp = self.sess.source_map().start_point(self.token.span); + if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow().get(&sp) { + self.sess.expr_parentheses_needed(&mut err, *sp, None); + } return Err(err); } } } } - let pat = Pat { node: pat, span: lo.to(self.prev_span), id: ast::DUMMY_NODE_ID }; + let pat = P(Pat { node: pat, span: lo.to(self.prev_span), id: ast::DUMMY_NODE_ID }); let pat = self.maybe_recover_from_bad_qpath(pat, true)?; if !allow_range_pat { @@ -4782,7 +4096,7 @@ impl<'a> Parser<'a> { } } - Ok(P(pat)) + Ok(pat) } /// Parses `ident` or `ident @ pat`. @@ -4830,7 +4144,7 @@ impl<'a> Parser<'a> { let parser_snapshot_after_type = self.clone(); mem::replace(self, parser_snapshot_before_type); - let snippet = self.sess.source_map().span_to_snippet(pat.span).unwrap(); + let snippet = self.span_to_snippet(pat.span).unwrap(); err.span_label(pat.span, format!("while parsing the type for `{}`", snippet)); (Some((parser_snapshot_after_type, colon_sp, err)), None) } @@ -4872,7 +4186,7 @@ impl<'a> Parser<'a> { } }; let hi = if self.token == token::Semi { - self.span + self.token.span } else { self.prev_span }; @@ -4925,92 +4239,6 @@ impl<'a> Parser<'a> { Ok(self.parse_stmt_(true)) } - // Eat tokens until we can be relatively sure we reached the end of the - // statement. This is something of a best-effort heuristic. - // - // We terminate when we find an unmatched `}` (without consuming it). - fn recover_stmt(&mut self) { - self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore) - } - - // If `break_on_semi` is `Break`, then we will stop consuming tokens after - // finding (and consuming) a `;` outside of `{}` or `[]` (note that this is - // approximate - it can mean we break too early due to macros, but that - // should only lead to sub-optimal recovery, not inaccurate parsing). - // - // If `break_on_block` is `Break`, then we will stop consuming tokens - // after finding (and consuming) a brace-delimited block. - fn recover_stmt_(&mut self, break_on_semi: SemiColonMode, break_on_block: BlockMode) { - let mut brace_depth = 0; - let mut bracket_depth = 0; - let mut in_block = false; - debug!("recover_stmt_ enter loop (semi={:?}, block={:?})", - break_on_semi, break_on_block); - loop { - debug!("recover_stmt_ loop {:?}", self.token); - match self.token { - token::OpenDelim(token::DelimToken::Brace) => { - brace_depth += 1; - self.bump(); - if break_on_block == BlockMode::Break && - brace_depth == 1 && - bracket_depth == 0 { - in_block = true; - } - } - token::OpenDelim(token::DelimToken::Bracket) => { - bracket_depth += 1; - self.bump(); - } - token::CloseDelim(token::DelimToken::Brace) => { - if brace_depth == 0 { - debug!("recover_stmt_ return - close delim {:?}", self.token); - break; - } - brace_depth -= 1; - self.bump(); - if in_block && bracket_depth == 0 && brace_depth == 0 { - debug!("recover_stmt_ return - block end {:?}", self.token); - break; - } - } - token::CloseDelim(token::DelimToken::Bracket) => { - bracket_depth -= 1; - if bracket_depth < 0 { - bracket_depth = 0; - } - self.bump(); - } - token::Eof => { - debug!("recover_stmt_ return - Eof"); - break; - } - token::Semi => { - self.bump(); - if break_on_semi == SemiColonMode::Break && - brace_depth == 0 && - bracket_depth == 0 { - debug!("recover_stmt_ return - Semi"); - break; - } - } - token::Comma => { - if break_on_semi == SemiColonMode::Comma && - brace_depth == 0 && - bracket_depth == 0 { - debug!("recover_stmt_ return - Semi"); - break; - } else { - self.bump(); - } - } - _ => { - self.bump() - } - } - } - } - fn parse_stmt_(&mut self, macro_legacy_warnings: bool) -> Option<Stmt> { self.parse_stmt_without_recovery(macro_legacy_warnings).unwrap_or_else(|mut e| { e.emit(); @@ -5019,11 +4247,11 @@ impl<'a> Parser<'a> { }) } - fn is_async_block(&mut self) -> bool { - self.token.is_keyword(keywords::Async) && + fn is_async_block(&self) -> bool { + self.token.is_keyword(kw::Async) && ( ( // `async move {` - self.look_ahead(1, |t| t.is_keyword(keywords::Move)) && + self.is_keyword_ahead(1, &[kw::Move]) && self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace)) ) || ( // `async {` self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace)) @@ -5031,117 +4259,107 @@ impl<'a> Parser<'a> { ) } - fn is_async_fn(&mut self) -> bool { - self.token.is_keyword(keywords::Async) && - self.look_ahead(1, |t| t.is_keyword(keywords::Fn)) + fn is_async_fn(&self) -> bool { + self.token.is_keyword(kw::Async) && + self.is_keyword_ahead(1, &[kw::Fn]) } - fn is_do_catch_block(&mut self) -> bool { - self.token.is_keyword(keywords::Do) && - self.look_ahead(1, |t| t.is_keyword(keywords::Catch)) && + fn is_do_catch_block(&self) -> bool { + self.token.is_keyword(kw::Do) && + self.is_keyword_ahead(1, &[kw::Catch]) && self.look_ahead(2, |t| *t == token::OpenDelim(token::Brace)) && !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL) } - fn is_try_block(&mut self) -> bool { - self.token.is_keyword(keywords::Try) && + fn is_try_block(&self) -> bool { + self.token.is_keyword(kw::Try) && self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace)) && - self.span.rust_2018() && + self.token.span.rust_2018() && // prevent `while try {} {}`, `if try {} {} else {}`, etc. !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL) } fn is_union_item(&self) -> bool { - self.token.is_keyword(keywords::Union) && + self.token.is_keyword(kw::Union) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident()) } fn is_crate_vis(&self) -> bool { - self.token.is_keyword(keywords::Crate) && self.look_ahead(1, |t| t != &token::ModSep) - } - - fn is_existential_type_decl(&self) -> bool { - self.token.is_keyword(keywords::Existential) && - self.look_ahead(1, |t| t.is_keyword(keywords::Type)) + self.token.is_keyword(kw::Crate) && self.look_ahead(1, |t| t != &token::ModSep) } - fn is_auto_trait_item(&mut self) -> bool { + fn is_auto_trait_item(&self) -> bool { // auto trait - (self.token.is_keyword(keywords::Auto) - && self.look_ahead(1, |t| t.is_keyword(keywords::Trait))) + (self.token.is_keyword(kw::Auto) && + self.is_keyword_ahead(1, &[kw::Trait])) || // unsafe auto trait - (self.token.is_keyword(keywords::Unsafe) && - self.look_ahead(1, |t| t.is_keyword(keywords::Auto)) && - self.look_ahead(2, |t| t.is_keyword(keywords::Trait))) + (self.token.is_keyword(kw::Unsafe) && + self.is_keyword_ahead(1, &[kw::Auto]) && + self.is_keyword_ahead(2, &[kw::Trait])) } fn eat_macro_def(&mut self, attrs: &[Attribute], vis: &Visibility, lo: Span) -> PResult<'a, Option<P<Item>>> { - let token_lo = self.span; - let (ident, def) = match self.token { - token::Ident(ident, false) if ident.name == keywords::Macro.name() => { - self.bump(); - let ident = self.parse_ident()?; - let tokens = if self.check(&token::OpenDelim(token::Brace)) { - match self.parse_token_tree() { - TokenTree::Delimited(_, _, tts) => tts, - _ => unreachable!(), - } - } else if self.check(&token::OpenDelim(token::Paren)) { - let args = self.parse_token_tree(); - let body = if self.check(&token::OpenDelim(token::Brace)) { - self.parse_token_tree() - } else { - self.unexpected()?; - unreachable!() - }; - TokenStream::new(vec![ - args.into(), - TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(), - body.into(), - ]) + let token_lo = self.token.span; + let (ident, def) = if self.eat_keyword(kw::Macro) { + let ident = self.parse_ident()?; + let tokens = if self.check(&token::OpenDelim(token::Brace)) { + match self.parse_token_tree() { + TokenTree::Delimited(_, _, tts) => tts, + _ => unreachable!(), + } + } else if self.check(&token::OpenDelim(token::Paren)) { + let args = self.parse_token_tree(); + let body = if self.check(&token::OpenDelim(token::Brace)) { + self.parse_token_tree() } else { self.unexpected()?; unreachable!() }; + TokenStream::new(vec![ + args.into(), + TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(), + body.into(), + ]) + } else { + self.unexpected()?; + unreachable!() + }; - (ident, ast::MacroDef { tokens: tokens.into(), legacy: false }) - } - token::Ident(ident, _) if ident.name == "macro_rules" && - self.look_ahead(1, |t| *t == token::Not) => { - let prev_span = self.prev_span; - self.complain_if_pub_macro(&vis.node, prev_span); - self.bump(); - self.bump(); - - let ident = self.parse_ident()?; - let (delim, tokens) = self.expect_delimited_token_tree()?; - if delim != MacDelimiter::Brace { - if !self.eat(&token::Semi) { - let msg = "macros that expand to items must either \ - be surrounded with braces or followed by a semicolon"; - self.span_err(self.prev_span, msg); - } - } + (ident, ast::MacroDef { tokens: tokens.into(), legacy: false }) + } else if self.check_keyword(sym::macro_rules) && + self.look_ahead(1, |t| *t == token::Not) && + self.look_ahead(2, |t| t.is_ident()) { + let prev_span = self.prev_span; + self.complain_if_pub_macro(&vis.node, prev_span); + self.bump(); + self.bump(); - (ident, ast::MacroDef { tokens: tokens, legacy: true }) + let ident = self.parse_ident()?; + let (delim, tokens) = self.expect_delimited_token_tree()?; + if delim != MacDelimiter::Brace && !self.eat(&token::Semi) { + self.report_invalid_macro_expansion_item(); } - _ => return Ok(None), + + (ident, ast::MacroDef { tokens, legacy: true }) + } else { + return Ok(None); }; let span = lo.to(self.prev_span); Ok(Some(self.mk_item(span, ident, ItemKind::MacroDef(def), vis.clone(), attrs.to_vec()))) } - fn parse_stmt_without_recovery(&mut self, - macro_legacy_warnings: bool) - -> PResult<'a, Option<Stmt>> { + fn parse_stmt_without_recovery( + &mut self, + macro_legacy_warnings: bool, + ) -> PResult<'a, Option<Stmt>> { maybe_whole!(self, NtStmt, |x| Some(x)); let attrs = self.parse_outer_attributes()?; - let lo = self.span; + let lo = self.token.span; - Ok(Some(if self.eat_keyword(keywords::Let) { + Ok(Some(if self.eat_keyword(kw::Let) { Stmt { id: ast::DUMMY_NODE_ID, node: StmtKind::Local(self.parse_local(attrs.into())?), @@ -5167,17 +4385,16 @@ impl<'a> Parser<'a> { !self.token.is_qpath_start() && !self.is_union_item() && !self.is_crate_vis() && - !self.is_existential_type_decl() && !self.is_auto_trait_item() && !self.is_async_fn() { - let pth = self.parse_path(PathStyle::Expr)?; + let path = self.parse_path(PathStyle::Expr)?; if !self.eat(&token::Not) { let expr = if self.check(&token::OpenDelim(token::Brace)) { - self.parse_struct_expr(lo, pth, ThinVec::new())? + self.parse_struct_expr(lo, path, ThinVec::new())? } else { let hi = self.prev_span; - self.mk_expr(lo.to(hi), ExprKind::Path(None, pth), ThinVec::new()) + self.mk_expr(lo.to(hi), ExprKind::Path(None, path), ThinVec::new()) }; let expr = self.with_res(Restrictions::STMT_EXPR, |this| { @@ -5192,34 +4409,6 @@ impl<'a> Parser<'a> { })); } - // it's a macro invocation - let id = match self.token { - token::OpenDelim(_) => keywords::Invalid.ident(), // no special identifier - _ => self.parse_ident()?, - }; - - // check that we're pointing at delimiters (need to check - // again after the `if`, because of `parse_ident` - // consuming more tokens). - match self.token { - token::OpenDelim(_) => {} - _ => { - // we only expect an ident if we didn't parse one - // above. - let ident_str = if id.name == keywords::Invalid.name() { - "identifier, " - } else { - "" - }; - let tok_str = self.this_token_descr(); - let mut err = self.fatal(&format!("expected {}`(` or `{{`, found {}", - ident_str, - tok_str)); - err.span_label(self.span, format!("expected {}`(` or `{{`", ident_str)); - return Err(err) - }, - } - let (delim, tts) = self.expect_delimited_token_tree()?; let hi = self.prev_span; @@ -5229,61 +4418,43 @@ impl<'a> Parser<'a> { MacStmtStyle::NoBraces }; - if id.name == keywords::Invalid.name() { - let mac = respan(lo.to(hi), Mac_ { path: pth, tts, delim }); - let node = if delim == MacDelimiter::Brace || - self.token == token::Semi || self.token == token::Eof { - StmtKind::Mac(P((mac, style, attrs.into()))) - } - // We used to incorrectly stop parsing macro-expanded statements here. - // If the next token will be an error anyway but could have parsed with the - // earlier behavior, stop parsing here and emit a warning to avoid breakage. - else if macro_legacy_warnings && self.token.can_begin_expr() && match self.token { - // These can continue an expression, so we can't stop parsing and warn. - token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | - token::BinOp(token::Minus) | token::BinOp(token::Star) | - token::BinOp(token::And) | token::BinOp(token::Or) | - token::AndAnd | token::OrOr | - token::DotDot | token::DotDotDot | token::DotDotEq => false, - _ => true, - } { - self.warn_missing_semicolon(); - StmtKind::Mac(P((mac, style, attrs.into()))) - } else { - let e = self.mk_mac_expr(lo.to(hi), mac.node, ThinVec::new()); - let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; - let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; - StmtKind::Expr(e) - }; - Stmt { - id: ast::DUMMY_NODE_ID, - span: lo.to(hi), - node, - } + let mac = respan(lo.to(hi), Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + }); + let node = if delim == MacDelimiter::Brace || + self.token == token::Semi || self.token == token::Eof { + StmtKind::Mac(P((mac, style, attrs.into()))) + } + // We used to incorrectly stop parsing macro-expanded statements here. + // If the next token will be an error anyway but could have parsed with the + // earlier behavior, stop parsing here and emit a warning to avoid breakage. + else if macro_legacy_warnings && + self.token.can_begin_expr() && + match self.token.kind { + // These can continue an expression, so we can't stop parsing and warn. + token::OpenDelim(token::Paren) | token::OpenDelim(token::Bracket) | + token::BinOp(token::Minus) | token::BinOp(token::Star) | + token::BinOp(token::And) | token::BinOp(token::Or) | + token::AndAnd | token::OrOr | + token::DotDot | token::DotDotDot | token::DotDotEq => false, + _ => true, + } { + self.warn_missing_semicolon(); + StmtKind::Mac(P((mac, style, attrs.into()))) } else { - // if it has a special ident, it's definitely an item - // - // Require a semicolon or braces. - if style != MacStmtStyle::Braces { - if !self.eat(&token::Semi) { - self.span_err(self.prev_span, - "macros that expand to items must \ - either be surrounded with braces or \ - followed by a semicolon"); - } - } - let span = lo.to(hi); - Stmt { - id: ast::DUMMY_NODE_ID, - span, - node: StmtKind::Item({ - self.mk_item( - span, id /*id is good here*/, - ItemKind::Mac(respan(span, Mac_ { path: pth, tts, delim })), - respan(lo, VisibilityKind::Inherited), - attrs) - }), - } + let e = self.mk_expr(mac.span, ExprKind::Mac(mac), ThinVec::new()); + let e = self.maybe_recover_from_bad_qpath(e, true)?; + let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; + let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; + StmtKind::Expr(e) + }; + Stmt { + id: ast::DUMMY_NODE_ID, + span: lo.to(hi), + node, } } else { // FIXME: Bad copy of attrs @@ -5304,7 +4475,9 @@ impl<'a> Parser<'a> { if s.prev_token_kind == PrevTokenKind::DocComment { s.span_fatal_err(s.prev_span, Error::UselessDocComment).emit(); } else if attrs.iter().any(|a| a.style == AttrStyle::Outer) { - s.span_err(s.span, "expected statement after outer attribute"); + s.span_err( + s.token.span, "expected statement after outer attribute" + ); } } }; @@ -5335,7 +4508,7 @@ impl<'a> Parser<'a> { } /// Checks if this expression is a successfully parsed statement. - fn expr_is_complete(&mut self, e: &Expr) -> bool { + fn expr_is_complete(&self, e: &Expr) -> bool { self.restrictions.contains(Restrictions::STMT_EXPR) && !classify::expr_requires_semi_to_be_stmt(e) } @@ -5344,26 +4517,26 @@ impl<'a> Parser<'a> { pub fn parse_block(&mut self) -> PResult<'a, P<Block>> { maybe_whole!(self, NtBlock, |x| x); - let lo = self.span; + let lo = self.token.span; if !self.eat(&token::OpenDelim(token::Brace)) { - let sp = self.span; + let sp = self.token.span; let tok = self.this_token_descr(); let mut e = self.span_fatal(sp, &format!("expected `{{`, found {}", tok)); let do_not_suggest_help = - self.token.is_keyword(keywords::In) || self.token == token::Colon; + self.token.is_keyword(kw::In) || self.token == token::Colon; - if self.token.is_ident_named("and") { + if self.token.is_ident_named(sym::and) { e.span_suggestion_short( - self.span, + self.token.span, "use `&&` instead of `and` for the boolean operator", "&&".to_string(), Applicability::MaybeIncorrect, ); } - if self.token.is_ident_named("or") { + if self.token.is_ident_named(sym::or) { e.span_suggestion_short( - self.span, + self.token.span, "use `||` instead of `or` for the boolean operator", "||".to_string(), Applicability::MaybeIncorrect, @@ -5390,20 +4563,15 @@ impl<'a> Parser<'a> { if self.eat(&token::Semi) { stmt_span = stmt_span.with_hi(self.prev_span.hi()); } - let sugg = pprust::to_string(|s| { - use crate::print::pprust::{PrintState, INDENT_UNIT}; - s.ibox(INDENT_UNIT)?; - s.bopen()?; - s.print_stmt(&stmt)?; - s.bclose_maybe_open(stmt.span, INDENT_UNIT, false) - }); - e.span_suggestion( - stmt_span, - "try placing this code inside a block", - sugg, - // speculative, has been misleading in the past (closed Issue #46836) - Applicability::MaybeIncorrect - ); + if let Ok(snippet) = self.span_to_snippet(stmt_span) { + e.span_suggestion( + stmt_span, + "try placing this code inside a block", + format!("{{ {} }}", snippet), + // speculative, has been misleading in the past (#46836) + Applicability::MaybeIncorrect, + ); + } } Err(mut e) => { self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore); @@ -5419,10 +4587,10 @@ impl<'a> Parser<'a> { } /// Parses a block. Inner attributes are allowed. - fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> { + crate fn parse_inner_attrs_and_block(&mut self) -> PResult<'a, (Vec<Attribute>, P<Block>)> { maybe_whole!(self, NtBlock, |x| (Vec::new(), x)); - let lo = self.span; + let lo = self.token.span; self.expect(&token::OpenDelim(token::Brace))?; Ok((self.parse_inner_attributes()?, self.parse_block_tail(lo, BlockCheckMode::Default)?)) @@ -5433,22 +4601,23 @@ impl<'a> Parser<'a> { fn parse_block_tail(&mut self, lo: Span, s: BlockCheckMode) -> PResult<'a, P<Block>> { let mut stmts = vec![]; while !self.eat(&token::CloseDelim(token::Brace)) { + if self.token == token::Eof { + break; + } let stmt = match self.parse_full_stmt(false) { Err(mut err) => { err.emit(); self.recover_stmt_(SemiColonMode::Ignore, BlockMode::Ignore); Some(Stmt { id: ast::DUMMY_NODE_ID, - node: StmtKind::Expr(DummyResult::raw_expr(self.span, true)), - span: self.span, + node: StmtKind::Expr(DummyResult::raw_expr(self.token.span, true)), + span: self.token.span, }) } Ok(stmt) => stmt, }; if let Some(stmt) = stmt { stmts.push(stmt); - } else if self.token == token::Eof { - break; } else { // Found only `;` or `}`. continue; @@ -5482,6 +4651,9 @@ impl<'a> Parser<'a> { { e.emit(); self.recover_stmt(); + // Don't complain about type errors in body tail after parse error (#57383). + let sp = expr.span.to(self.prev_span); + stmt.node = StmtKind::Expr(DummyResult::raw_expr(sp, true)); } } } @@ -5499,13 +4671,12 @@ impl<'a> Parser<'a> { if self.eat(&token::Semi) { stmt = stmt.add_trailing_semicolon(); } - - stmt.span = stmt.span.with_hi(self.prev_span.hi()); + stmt.span = stmt.span.to(self.prev_span); Ok(Some(stmt)) } fn warn_missing_semicolon(&self) { - self.diagnostic().struct_span_warn(self.span, { + self.diagnostic().struct_span_warn(self.token.span, { &format!("expected `;`, found {}", self.this_token_descr()) }).note({ "This was erroneously allowed and will become a hard error in a future release" @@ -5540,16 +4711,16 @@ impl<'a> Parser<'a> { let mut last_plus_span = None; let mut was_negative = false; loop { - // This needs to be synchronized with `Token::can_begin_bound`. + // This needs to be synchronized with `TokenKind::can_begin_bound`. let is_bound_start = self.check_path() || self.check_lifetime() || self.check(&token::Not) || // used for error reporting only self.check(&token::Question) || - self.check_keyword(keywords::For) || + self.check_keyword(kw::For) || self.check(&token::OpenDelim(token::Paren)); if is_bound_start { - let lo = self.span; + let lo = self.token.span; let has_parens = self.eat(&token::OpenDelim(token::Paren)); - let inner_lo = self.span; + let inner_lo = self.token.span; let is_negative = self.eat(&token::Not); let question = if self.eat(&token::Question) { Some(self.prev_span) } else { None }; if self.token.is_lifetime() { @@ -5565,7 +4736,7 @@ impl<'a> Parser<'a> { lo.to(self.prev_span), "parenthesized lifetime bounds are not supported" ); - if let Ok(snippet) = self.sess.source_map().span_to_snippet(inner_span) { + if let Ok(snippet) = self.span_to_snippet(inner_span) { err.span_suggestion_short( lo.to(self.prev_span), "remove the parentheses", @@ -5610,25 +4781,32 @@ impl<'a> Parser<'a> { if !negative_bounds.is_empty() || was_negative { let plural = negative_bounds.len() > 1; - let mut err = self.struct_span_err(negative_bounds, - "negative trait bounds are not supported"); + let last_span = negative_bounds.last().map(|sp| *sp); + let mut err = self.struct_span_err( + negative_bounds, + "negative trait bounds are not supported", + ); + if let Some(sp) = last_span { + err.span_label(sp, "negative trait bounds are not supported"); + } if let Some(bound_list) = colon_span { let bound_list = bound_list.to(self.prev_span); let mut new_bound_list = String::new(); if !bounds.is_empty() { let mut snippets = bounds.iter().map(|bound| bound.span()) - .map(|span| self.sess.source_map().span_to_snippet(span)); + .map(|span| self.span_to_snippet(span)); while let Some(Ok(snippet)) = snippets.next() { new_bound_list.push_str(" + "); new_bound_list.push_str(&snippet); } new_bound_list = new_bound_list.replacen(" +", ":", 1); } - err.span_suggestion_short(bound_list, - &format!("remove the trait bound{}", - if plural { "s" } else { "" }), - new_bound_list, - Applicability::MachineApplicable); + err.span_suggestion_hidden( + bound_list, + &format!("remove the trait bound{}", if plural { "s" } else { "" }), + new_bound_list, + Applicability::MachineApplicable, + ); } err.emit(); } @@ -5636,7 +4814,8 @@ impl<'a> Parser<'a> { return Ok(bounds); } - fn parse_generic_bounds(&mut self, colon_span: Option<Span>) -> PResult<'a, GenericBounds> { + crate fn parse_generic_bounds(&mut self, + colon_span: Option<Span>) -> PResult<'a, GenericBounds> { self.parse_generic_bounds_common(true, colon_span) } @@ -5714,7 +4893,7 @@ impl<'a> Parser<'a> { } fn parse_const_param(&mut self, preceding_attrs: Vec<Attribute>) -> PResult<'a, GenericParam> { - self.expect_keyword(keywords::Const)?; + self.expect_keyword(kw::Const)?; let ident = self.parse_ident()?; self.expect(&token::Colon)?; let ty = self.parse_ty()?; @@ -5751,7 +4930,7 @@ impl<'a> Parser<'a> { bounds, kind: ast::GenericParamKind::Lifetime, }); - } else if self.check_keyword(keywords::Const) { + } else if self.check_keyword(kw::Const) { // Parse const parameter. params.push(self.parse_const_param(attrs)?); } else if self.check_ident() { @@ -5797,24 +4976,22 @@ impl<'a> Parser<'a> { /// | ( < lifetimes , typaramseq ( , )? > ) /// where typaramseq = ( typaram ) | ( typaram , typaramseq ) fn parse_generics(&mut self) -> PResult<'a, ast::Generics> { - maybe_whole!(self, NtGenerics, |x| x); - - let span_lo = self.span; - if self.eat_lt() { + let span_lo = self.token.span; + let (params, span) = if self.eat_lt() { let params = self.parse_generic_params()?; self.expect_gt()?; - Ok(ast::Generics { - params, - where_clause: WhereClause { - id: ast::DUMMY_NODE_ID, - predicates: Vec::new(), - span: syntax_pos::DUMMY_SP, - }, - span: span_lo.to(self.prev_span), - }) + (params, span_lo.to(self.prev_span)) } else { - Ok(ast::Generics::default()) - } + (vec![], self.prev_span.between(self.token.span)) + }; + Ok(ast::Generics { + params, + where_clause: WhereClause { + predicates: Vec::new(), + span: DUMMY_SP, + }, + span, + }) } /// Parses generic args (within a path segment) with recovery for extra leading angle brackets. @@ -5830,7 +5007,7 @@ impl<'a> Parser<'a> { &mut self, style: PathStyle, lo: Span, - ) -> PResult<'a, (Vec<GenericArg>, Vec<TypeBinding>)> { + ) -> PResult<'a, (Vec<GenericArg>, Vec<AssocTyConstraint>)> { // We need to detect whether there are extra leading left angle brackets and produce an // appropriate error and suggestion. This cannot be implemented by looking ahead at // upcoming tokens for a matching `>` character - if there are unmatched `<` tokens @@ -5965,51 +5142,61 @@ impl<'a> Parser<'a> { /// Parses (possibly empty) list of lifetime and type arguments and associated type bindings, /// possibly including trailing comma. - fn parse_generic_args(&mut self) -> PResult<'a, (Vec<GenericArg>, Vec<TypeBinding>)> { + fn parse_generic_args(&mut self) -> PResult<'a, (Vec<GenericArg>, Vec<AssocTyConstraint>)> { let mut args = Vec::new(); - let mut bindings = Vec::new(); - let mut misplaced_assoc_ty_bindings: Vec<Span> = Vec::new(); - let mut assoc_ty_bindings: Vec<Span> = Vec::new(); + let mut constraints = Vec::new(); + let mut misplaced_assoc_ty_constraints: Vec<Span> = Vec::new(); + let mut assoc_ty_constraints: Vec<Span> = Vec::new(); - let args_lo = self.span; + let args_lo = self.token.span; loop { if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) { // Parse lifetime argument. args.push(GenericArg::Lifetime(self.expect_lifetime())); - misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); - } else if self.check_ident() && self.look_ahead(1, |t| t == &token::Eq) { - // Parse associated type binding. - let lo = self.span; + misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints); + } else if self.check_ident() && self.look_ahead(1, + |t| t == &token::Eq || t == &token::Colon) { + // Parse associated type constraint. + let lo = self.token.span; let ident = self.parse_ident()?; - self.bump(); - let ty = self.parse_ty()?; + let kind = if self.eat(&token::Eq) { + AssocTyConstraintKind::Equality { + ty: self.parse_ty()?, + } + } else if self.eat(&token::Colon) { + AssocTyConstraintKind::Bound { + bounds: self.parse_generic_bounds(Some(self.prev_span))?, + } + } else { + unreachable!(); + }; let span = lo.to(self.prev_span); - bindings.push(TypeBinding { + constraints.push(AssocTyConstraint { id: ast::DUMMY_NODE_ID, ident, - ty, + kind, span, }); - assoc_ty_bindings.push(span); + assoc_ty_constraints.push(span); } else if self.check_const_arg() { - // FIXME(const_generics): to distinguish between idents for types and consts, - // we should introduce a GenericArg::Ident in the AST and distinguish when - // lowering to the HIR. For now, idents for const args are not permitted. - // Parse const argument. - let expr = if let token::OpenDelim(token::Brace) = self.token { - self.parse_block_expr(None, self.span, BlockCheckMode::Default, ThinVec::new())? + let expr = if let token::OpenDelim(token::Brace) = self.token.kind { + self.parse_block_expr( + None, self.token.span, BlockCheckMode::Default, ThinVec::new() + )? } else if self.token.is_ident() { // FIXME(const_generics): to distinguish between idents for types and consts, // we should introduce a GenericArg::Ident in the AST and distinguish when // lowering to the HIR. For now, idents for const args are not permitted. - return Err( - self.fatal("identifiers may currently not be used for const generics") - ); + if self.token.is_keyword(kw::True) || self.token.is_keyword(kw::False) { + self.parse_literal_maybe_minus()? + } else { + return Err( + self.fatal("identifiers may currently not be used for const generics") + ); + } } else { - // FIXME(const_generics): this currently conflicts with emplacement syntax - // with negative integer literals. self.parse_literal_maybe_minus()? }; let value = AnonConst { @@ -6017,11 +5204,11 @@ impl<'a> Parser<'a> { value: expr, }; args.push(GenericArg::Const(value)); - misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); + misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints); } else if self.check_type() { // Parse type argument. args.push(GenericArg::Type(self.parse_ty()?)); - misplaced_assoc_ty_bindings.append(&mut assoc_ty_bindings); + misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints); } else { break } @@ -6034,12 +5221,12 @@ impl<'a> Parser<'a> { // FIXME: we would like to report this in ast_validation instead, but we currently do not // preserve ordering of generic parameters with respect to associated type binding, so we // lose that information after parsing. - if misplaced_assoc_ty_bindings.len() > 0 { + if misplaced_assoc_ty_constraints.len() > 0 { let mut err = self.struct_span_err( args_lo.to(self.prev_span), "associated type bindings must be declared after generic parameters", ); - for span in misplaced_assoc_ty_bindings { + for span in misplaced_assoc_ty_constraints { err.span_label( span, "this associated type binding should be moved after the generic parameters", @@ -6048,7 +5235,7 @@ impl<'a> Parser<'a> { err.emit(); } - Ok((args, bindings)) + Ok((args, constraints)) } /// Parses an optional where-clause and places it in `generics`. @@ -6057,15 +5244,12 @@ impl<'a> Parser<'a> { /// where T : Trait<U, V> + 'b, 'a : 'b /// ``` fn parse_where_clause(&mut self) -> PResult<'a, WhereClause> { - maybe_whole!(self, NtWhereClause, |x| x); - let mut where_clause = WhereClause { - id: ast::DUMMY_NODE_ID, predicates: Vec::new(), - span: syntax_pos::DUMMY_SP, + span: self.prev_span.to(self.prev_span), }; - if !self.eat_keyword(keywords::Where) { + if !self.eat_keyword(kw::Where) { return Ok(where_clause); } let lo = self.prev_span; @@ -6084,7 +5268,7 @@ impl<'a> Parser<'a> { } loop { - let lo = self.span; + let lo = self.token.span; if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) { let lifetime = self.expect_lifetime(); // Bounds starting with a colon are mandatory, but possibly empty. @@ -6101,9 +5285,10 @@ impl<'a> Parser<'a> { // Parse optional `for<'a, 'b>`. // This `for` is parsed greedily and applies to the whole predicate, // the bounded type can have its own `for` applying only to it. - // Example 1: for<'a> Trait1<'a>: Trait2<'a /*ok*/> - // Example 2: (for<'a> Trait1<'a>): Trait2<'a /*not ok*/> - // Example 3: for<'a> for<'b> Trait1<'a, 'b>: Trait2<'a /*ok*/, 'b /*not ok*/> + // Examples: + // * `for<'a> Trait1<'a>: Trait2<'a /* ok */>` + // * `(for<'a> Trait1<'a>): Trait2<'a /* not ok */>` + // * `for<'a> for<'b> Trait1<'a, 'b>: Trait2<'a /* ok */, 'b /* not ok */>` let lifetime_defs = self.parse_late_bound_lifetime_defs()?; // Parse type with mandatory colon and (possibly empty) bounds, @@ -6149,55 +5334,48 @@ impl<'a> Parser<'a> { fn parse_fn_args(&mut self, named_args: bool, allow_c_variadic: bool) -> PResult<'a, (Vec<Arg> , bool)> { - self.expect(&token::OpenDelim(token::Paren))?; - - let sp = self.span; + let sp = self.token.span; let mut c_variadic = false; - let (args, recovered): (Vec<Option<Arg>>, bool) = - self.parse_seq_to_before_end( - &token::CloseDelim(token::Paren), - SeqSep::trailing_allowed(token::Comma), - |p| { - // If the argument is a C-variadic argument we should not - // enforce named arguments. - let enforce_named_args = if p.token == token::DotDotDot { + let (args, _): (Vec<Option<Arg>>, _) = self.parse_paren_comma_seq(|p| { + let do_not_enforce_named_arguments_for_c_variadic = + |token: &token::Token| -> bool { + if token == &token::DotDotDot { false } else { named_args - }; - match p.parse_arg_general(enforce_named_args, false, - allow_c_variadic) { - Ok(arg) => { - if let TyKind::CVarArgs = arg.ty.node { - c_variadic = true; - if p.token != token::CloseDelim(token::Paren) { - let span = p.span; - p.span_err(span, - "`...` must be the last argument of a C-variadic function"); - Ok(None) - } else { - Ok(Some(arg)) - } - } else { - Ok(Some(arg)) - } - }, - Err(mut e) => { - e.emit(); - let lo = p.prev_span; - // Skip every token until next possible arg or end. - p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); - // Create a placeholder argument for proper arg count (issue #34264). - let span = lo.to(p.prev_span); - Ok(Some(dummy_arg(span))) + } + }; + match p.parse_arg_general( + false, + allow_c_variadic, + do_not_enforce_named_arguments_for_c_variadic + ) { + Ok(arg) => { + if let TyKind::CVarArgs = arg.ty.node { + c_variadic = true; + if p.token != token::CloseDelim(token::Paren) { + let span = p.token.span; + p.span_err(span, + "`...` must be the last argument of a C-variadic function"); + Ok(None) + } else { + Ok(Some(arg)) } + } else { + Ok(Some(arg)) } + }, + Err(mut e) => { + e.emit(); + let lo = p.prev_span; + // Skip every token until next possible arg or end. + p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(token::Paren)]); + // Create a placeholder argument for proper arg count (issue #34264). + let span = lo.to(p.prev_span); + Ok(Some(dummy_arg(Ident::new(kw::Invalid, span)))) } - )?; - - if !recovered { - self.eat(&token::CloseDelim(token::Paren)); - } + } + })?; let args: Vec<_> = args.into_iter().filter_map(|x| x).collect(); @@ -6211,7 +5389,6 @@ impl<'a> Parser<'a> { /// Parses the argument list and result type of a function declaration. fn parse_fn_decl(&mut self, allow_c_variadic: bool) -> PResult<'a, P<FnDecl>> { - let (args, c_variadic) = self.parse_fn_args(true, allow_c_variadic)?; let ret_ty = self.parse_ret_ty(true)?; @@ -6223,33 +5400,35 @@ impl<'a> Parser<'a> { } /// Returns the parsed optional self argument and whether a self shortcut was used. + /// + /// See `parse_self_arg_with_attrs` to collect attributes. fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> { - let expect_ident = |this: &mut Self| match this.token { + let expect_ident = |this: &mut Self| match this.token.kind { // Preserve hygienic context. - token::Ident(ident, _) => - { let span = this.span; this.bump(); Ident::new(ident.name, span) } + token::Ident(name, _) => + { let span = this.token.span; this.bump(); Ident::new(name, span) } _ => unreachable!() }; let isolated_self = |this: &mut Self, n| { - this.look_ahead(n, |t| t.is_keyword(keywords::SelfLower)) && + this.look_ahead(n, |t| t.is_keyword(kw::SelfLower)) && this.look_ahead(n + 1, |t| t != &token::ModSep) }; - // Parse optional self parameter of a method. - // Only a limited set of initial token sequences is considered self parameters, anything + // Parse optional `self` parameter of a method. + // Only a limited set of initial token sequences is considered `self` parameters; anything // else is parsed as a normal function parameter list, so some lookahead is required. - let eself_lo = self.span; - let (eself, eself_ident, eself_hi) = match self.token { + let eself_lo = self.token.span; + let (eself, eself_ident, eself_hi) = match self.token.kind { token::BinOp(token::And) => { - // &self - // &mut self - // &'lt self - // &'lt mut self - // ¬_self + // `&self` + // `&mut self` + // `&'lt self` + // `&'lt mut self` + // `¬_self` (if isolated_self(self, 1) { self.bump(); SelfKind::Region(None, Mutability::Immutable) - } else if self.look_ahead(1, |t| t.is_keyword(keywords::Mut)) && + } else if self.is_keyword_ahead(1, &[kw::Mut]) && isolated_self(self, 2) { self.bump(); self.bump(); @@ -6260,7 +5439,7 @@ impl<'a> Parser<'a> { let lt = self.expect_lifetime(); SelfKind::Region(Some(lt), Mutability::Immutable) } else if self.look_ahead(1, |t| t.is_lifetime()) && - self.look_ahead(2, |t| t.is_keyword(keywords::Mut)) && + self.is_keyword_ahead(2, &[kw::Mut]) && isolated_self(self, 3) { self.bump(); let lt = self.expect_lifetime(); @@ -6271,24 +5450,24 @@ impl<'a> Parser<'a> { }, expect_ident(self), self.prev_span) } token::BinOp(token::Star) => { - // *self - // *const self - // *mut self - // *not_self + // `*self` + // `*const self` + // `*mut self` + // `*not_self` // Emit special error for `self` cases. let msg = "cannot pass `self` by raw pointer"; (if isolated_self(self, 1) { self.bump(); - self.struct_span_err(self.span, msg) - .span_label(self.span, msg) + self.struct_span_err(self.token.span, msg) + .span_label(self.token.span, msg) .emit(); SelfKind::Value(Mutability::Immutable) } else if self.look_ahead(1, |t| t.is_mutability()) && isolated_self(self, 2) { self.bump(); self.bump(); - self.struct_span_err(self.span, msg) - .span_label(self.span, msg) + self.struct_span_err(self.token.span, msg) + .span_label(self.token.span, msg) .emit(); SelfKind::Value(Mutability::Immutable) } else { @@ -6297,8 +5476,8 @@ impl<'a> Parser<'a> { } token::Ident(..) => { if isolated_self(self, 0) { - // self - // self: TYPE + // `self` + // `self: TYPE` let eself_ident = expect_ident(self); let eself_hi = self.prev_span; (if self.eat(&token::Colon) { @@ -6307,10 +5486,10 @@ impl<'a> Parser<'a> { } else { SelfKind::Value(Mutability::Immutable) }, eself_ident, eself_hi) - } else if self.token.is_keyword(keywords::Mut) && + } else if self.token.is_keyword(kw::Mut) && isolated_self(self, 1) { - // mut self - // mut self: TYPE + // `mut self` + // `mut self: TYPE` self.bump(); let eself_ident = expect_ident(self); let eself_hi = self.prev_span; @@ -6328,7 +5507,18 @@ impl<'a> Parser<'a> { }; let eself = source_map::respan(eself_lo.to(eself_hi), eself); - Ok(Some(Arg::from_self(eself, eself_ident))) + Ok(Some(Arg::from_self(ThinVec::default(), eself, eself_ident))) + } + + /// Returns the parsed optional self argument with attributes and whether a self + /// shortcut was used. + fn parse_self_arg_with_attrs(&mut self) -> PResult<'a, Option<Arg>> { + let attrs = self.parse_arg_attributes()?; + let arg_opt = self.parse_self_arg()?; + Ok(arg_opt.map(|mut arg| { + arg.attrs = attrs.into(); + arg + })) } /// Parses the parameter list and result type of a function that may have a `self` parameter. @@ -6337,17 +5527,17 @@ impl<'a> Parser<'a> { { self.expect(&token::OpenDelim(token::Paren))?; - // Parse optional self argument - let self_arg = self.parse_self_arg()?; + // Parse optional self argument. + let self_arg = self.parse_self_arg_with_attrs()?; // Parse the rest of the function parameter list. let sep = SeqSep::trailing_allowed(token::Comma); - let (fn_inputs, recovered) = if let Some(self_arg) = self_arg { + let (mut fn_inputs, recovered) = if let Some(self_arg) = self_arg { if self.check(&token::CloseDelim(token::Paren)) { (vec![self_arg], false) } else if self.eat(&token::Comma) { let mut fn_inputs = vec![self_arg]; - let (mut input, recovered) = self.parse_seq_to_before_end( + let (mut input, _, recovered) = self.parse_seq_to_before_end( &token::CloseDelim(token::Paren), sep, parse_arg_fn)?; fn_inputs.append(&mut input); (fn_inputs, recovered) @@ -6358,13 +5548,18 @@ impl<'a> Parser<'a> { } } } else { - self.parse_seq_to_before_end(&token::CloseDelim(token::Paren), sep, parse_arg_fn)? + let (input, _, recovered) = + self.parse_seq_to_before_end(&token::CloseDelim(token::Paren), sep, parse_arg_fn)?; + (input, recovered) }; if !recovered { // Parse closing paren and return type. self.expect(&token::CloseDelim(token::Paren))?; } + // Replace duplicated recovered arguments with `_` pattern to avoid unecessary errors. + self.deduplicate_recovered_arg_names(&mut fn_inputs); + Ok(P(FnDecl { inputs: fn_inputs, output: self.parse_ret_ty(true)?, @@ -6405,7 +5600,7 @@ impl<'a> Parser<'a> { Ok((id, generics)) } - fn mk_item(&mut self, span: Span, ident: Ident, node: ItemKind, vis: Visibility, + fn mk_item(&self, span: Span, ident: Ident, node: ItemKind, vis: Visibility, attrs: Vec<Attribute>) -> P<Item> { P(Item { ident, @@ -6436,10 +5631,9 @@ impl<'a> Parser<'a> { /// Returns `true` if we are looking at `const ID` /// (returns `false` for things like `const fn`, etc.). - fn is_const_item(&mut self) -> bool { - self.token.is_keyword(keywords::Const) && - !self.look_ahead(1, |t| t.is_keyword(keywords::Fn)) && - !self.look_ahead(1, |t| t.is_keyword(keywords::Unsafe)) + fn is_const_item(&self) -> bool { + self.token.is_keyword(kw::Const) && + !self.is_keyword_ahead(1, &[kw::Fn, kw::Unsafe]) } /// Parses all the "front matter" for a `fn` declaration, up to @@ -6458,22 +5652,30 @@ impl<'a> Parser<'a> { Abi )> { - let is_const_fn = self.eat_keyword(keywords::Const); + let is_const_fn = self.eat_keyword(kw::Const); let const_span = self.prev_span; - let unsafety = self.parse_unsafety(); let asyncness = self.parse_asyncness(); + if let IsAsync::Async { .. } = asyncness { + self.ban_async_in_2015(self.prev_span); + } let asyncness = respan(self.prev_span, asyncness); + let unsafety = self.parse_unsafety(); let (constness, unsafety, abi) = if is_const_fn { (respan(const_span, Constness::Const), unsafety, Abi::Rust) } else { - let abi = if self.eat_keyword(keywords::Extern) { + let abi = if self.eat_keyword(kw::Extern) { self.parse_opt_abi()?.unwrap_or(Abi::C) } else { Abi::Rust }; (respan(self.prev_span, Constness::NotConst), unsafety, abi) }; - self.expect_keyword(keywords::Fn)?; + if !self.eat_keyword(kw::Fn) { + // It is possible for `expect_one_of` to recover given the contents of + // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't + // account for this. + if !self.expect_one_of(&[], &[])? { unreachable!() } + } Ok((constness, unsafety, asyncness, abi)) } @@ -6499,20 +5701,20 @@ impl<'a> Parser<'a> { fn parse_impl_item_(&mut self, at_end: &mut bool, mut attrs: Vec<Attribute>) -> PResult<'a, ImplItem> { - let lo = self.span; + let lo = self.token.span; let vis = self.parse_visibility(false)?; let defaultness = self.parse_defaultness(); let (name, node, generics) = if let Some(type_) = self.eat_type() { let (name, alias, generics) = type_?; let kind = match alias { - AliasKind::Weak(typ) => ast::ImplItemKind::Type(typ), - AliasKind::Existential(bounds) => ast::ImplItemKind::Existential(bounds), + AliasKind::Weak(typ) => ast::ImplItemKind::TyAlias(typ), + AliasKind::OpaqueTy(bounds) => ast::ImplItemKind::OpaqueTy(bounds), }; (name, kind, generics) } else if self.is_const_item() { // This parses the grammar: // ImplItemConst = "const" Ident ":" Ty "=" Expr ";" - self.expect_keyword(keywords::Const)?; + self.expect_keyword(kw::Const)?; let name = self.parse_ident()?; self.expect(&token::Colon)?; let typ = self.parse_ty()?; @@ -6539,15 +5741,11 @@ impl<'a> Parser<'a> { }) } - fn complain_if_pub_macro(&mut self, vis: &VisibilityKind, sp: Span) { + fn complain_if_pub_macro(&self, vis: &VisibilityKind, sp: Span) { match *vis { VisibilityKind::Inherited => {} _ => { - let is_macro_rules: bool = match self.token { - token::Ident(sid, _) => sid.name == Symbol::intern("macro_rules"), - _ => false, - }; - let mut err = if is_macro_rules { + let mut err = if self.token.is_keyword(sym::macro_rules) { let mut err = self.diagnostic() .struct_span_err(sp, "can't qualify macro_rules invocation with `pub`"); err.span_suggestion( @@ -6568,7 +5766,7 @@ impl<'a> Parser<'a> { } } - fn missing_assoc_item_kind_err(&mut self, item_type: &str, prev_span: Span) + fn missing_assoc_item_kind_err(&self, item_type: &str, prev_span: Span) -> DiagnosticBuilder<'a> { let expected_kinds = if item_type == "extern" { @@ -6602,13 +5800,15 @@ impl<'a> Parser<'a> { // code copied from parse_macro_use_or_failure... abstraction! if let Some(mac) = self.parse_assoc_macro_invoc("impl", Some(vis), at_end)? { // method macro - Ok((keywords::Invalid.ident(), vec![], ast::Generics::default(), + Ok((Ident::invalid(), vec![], ast::Generics::default(), ast::ImplItemKind::Macro(mac))) } else { let (constness, unsafety, asyncness, abi) = self.parse_fn_front_matter()?; let ident = self.parse_ident()?; let mut generics = self.parse_generics()?; - let decl = self.parse_fn_decl_with_self(|p| p.parse_arg())?; + let decl = self.parse_fn_decl_with_self(|p| { + p.parse_arg_general(true, false, |_| true) + })?; generics.where_clause = self.parse_where_clause()?; *at_end = true; let (inner_attrs, body) = self.parse_inner_attrs_and_block()?; @@ -6656,6 +5856,23 @@ impl<'a> Parser<'a> { self.expect(&token::OpenDelim(token::Brace))?; let mut trait_items = vec![]; while !self.eat(&token::CloseDelim(token::Brace)) { + if let token::DocComment(_) = self.token.kind { + if self.look_ahead(1, + |tok| tok == &token::CloseDelim(token::Brace)) { + self.diagnostic().struct_span_err_with_code( + self.token.span, + "found a documentation comment that doesn't document anything", + DiagnosticId::Error("E0584".into()), + ) + .help( + "doc comments must come before what they document, maybe a \ + comment was intended with `//`?", + ) + .emit(); + self.bump(); + continue; + } + } let mut at_end = false; match self.parse_trait_item(&mut at_end) { Ok(item) => trait_items.push(item), @@ -6693,7 +5910,7 @@ impl<'a> Parser<'a> { self.look_ahead(1, |t| t.is_lifetime() || t.is_ident()) && self.look_ahead(2, |t| t == &token::Gt || t == &token::Comma || t == &token::Colon || t == &token::Eq) || - self.look_ahead(1, |t| t.is_keyword(keywords::Const))) + self.is_keyword_ahead(1, &[kw::Const])) } fn parse_impl_body(&mut self) -> PResult<'a, (Vec<ImplItem>, Vec<Attribute>)> { @@ -6743,16 +5960,24 @@ impl<'a> Parser<'a> { }; // Parse both types and traits as a type, then reinterpret if necessary. - let ty_first = self.parse_ty()?; + let err_path = |span| ast::Path::from_ident(Ident::new(kw::Invalid, span)); + let ty_first = if self.token.is_keyword(kw::For) && + self.look_ahead(1, |t| t != &token::Lt) { + let span = self.prev_span.between(self.token.span); + self.struct_span_err(span, "missing trait in a trait impl").emit(); + P(Ty { node: TyKind::Path(None, err_path(span)), span, id: ast::DUMMY_NODE_ID }) + } else { + self.parse_ty()? + }; // If `for` is missing we try to recover. - let has_for = self.eat_keyword(keywords::For); - let missing_for_span = self.prev_span.between(self.span); + let has_for = self.eat_keyword(kw::For); + let missing_for_span = self.prev_span.between(self.token.span); let ty_second = if self.token == token::DotDot { // We need to report this error after `cfg` expansion for compatibility reasons self.bump(); // `..`, do not add it to expected tokens - Some(P(Ty { node: TyKind::Err, span: self.prev_span, id: ast::DUMMY_NODE_ID })) + Some(DummyResult::raw_ty(self.prev_span, true)) } else if has_for || self.token.can_begin_type() { Some(self.parse_ty()?) } else { @@ -6782,7 +6007,7 @@ impl<'a> Parser<'a> { TyKind::Path(None, path) => path, _ => { self.span_err(ty_first.span, "expected a trait, found type"); - ast::Path::from_ident(Ident::new(keywords::Invalid.name(), ty_first.span)) + err_path(ty_first.span) } }; let trait_ref = TraitRef { path, ref_id: ty_first.id }; @@ -6797,11 +6022,11 @@ impl<'a> Parser<'a> { } }; - Ok((keywords::Invalid.ident(), item_kind, Some(attrs))) + Ok((Ident::invalid(), item_kind, Some(attrs))) } fn parse_late_bound_lifetime_defs(&mut self) -> PResult<'a, Vec<GenericParam>> { - if self.eat_keyword(keywords::For) { + if self.eat_keyword(kw::For) { self.expect_lt()?; let params = self.parse_generic_params()?; self.expect_gt()?; @@ -6833,21 +6058,23 @@ impl<'a> Parser<'a> { // Otherwise if we look ahead and see a paren we parse a tuple-style // struct. - let vdata = if self.token.is_keyword(keywords::Where) { + let vdata = if self.token.is_keyword(kw::Where) { generics.where_clause = self.parse_where_clause()?; if self.eat(&token::Semi) { // If we see a: `struct Foo<T> where T: Copy;` style decl. VariantData::Unit(ast::DUMMY_NODE_ID) } else { // If we see: `struct Foo<T> where T: Copy { ... }` - VariantData::Struct(self.parse_record_struct_body()?, ast::DUMMY_NODE_ID) + let (fields, recovered) = self.parse_record_struct_body()?; + VariantData::Struct(fields, recovered) } // No `where` so: `struct Foo<T>;` } else if self.eat(&token::Semi) { VariantData::Unit(ast::DUMMY_NODE_ID) // Record-style struct definition } else if self.token == token::OpenDelim(token::Brace) { - VariantData::Struct(self.parse_record_struct_body()?, ast::DUMMY_NODE_ID) + let (fields, recovered) = self.parse_record_struct_body()?; + VariantData::Struct(fields, recovered) // Tuple-style struct definition with optional where-clause. } else if self.token == token::OpenDelim(token::Paren) { let body = VariantData::Tuple(self.parse_tuple_struct_body()?, ast::DUMMY_NODE_ID); @@ -6860,7 +6087,7 @@ impl<'a> Parser<'a> { "expected `where`, `{{`, `(`, or `;` after struct name, found {}", token_str )); - err.span_label(self.span, "expected `where`, `{`, `(`, or `;` after struct name"); + err.span_label(self.token.span, "expected `where`, `{`, `(`, or `;` after struct name"); return Err(err); }; @@ -6873,48 +6100,34 @@ impl<'a> Parser<'a> { let mut generics = self.parse_generics()?; - let vdata = if self.token.is_keyword(keywords::Where) { + let vdata = if self.token.is_keyword(kw::Where) { generics.where_clause = self.parse_where_clause()?; - VariantData::Struct(self.parse_record_struct_body()?, ast::DUMMY_NODE_ID) + let (fields, recovered) = self.parse_record_struct_body()?; + VariantData::Struct(fields, recovered) } else if self.token == token::OpenDelim(token::Brace) { - VariantData::Struct(self.parse_record_struct_body()?, ast::DUMMY_NODE_ID) + let (fields, recovered) = self.parse_record_struct_body()?; + VariantData::Struct(fields, recovered) } else { let token_str = self.this_token_descr(); let mut err = self.fatal(&format!( "expected `where` or `{{` after union name, found {}", token_str)); - err.span_label(self.span, "expected `where` or `{` after union name"); + err.span_label(self.token.span, "expected `where` or `{` after union name"); return Err(err); }; Ok((class_name, ItemKind::Union(vdata, generics), None)) } - fn consume_block(&mut self, delim: token::DelimToken) { - let mut brace_depth = 0; - loop { - if self.eat(&token::OpenDelim(delim)) { - brace_depth += 1; - } else if self.eat(&token::CloseDelim(delim)) { - if brace_depth == 0 { - return; - } else { - brace_depth -= 1; - continue; - } - } else if self.token == token::Eof || self.eat(&token::CloseDelim(token::NoDelim)) { - return; - } else { - self.bump(); - } - } - } - - fn parse_record_struct_body(&mut self) -> PResult<'a, Vec<StructField>> { + fn parse_record_struct_body( + &mut self, + ) -> PResult<'a, (Vec<StructField>, /* recovered */ bool)> { let mut fields = Vec::new(); + let mut recovered = false; if self.eat(&token::OpenDelim(token::Brace)) { while self.token != token::CloseDelim(token::Brace) { let field = self.parse_struct_decl_field().map_err(|e| { self.recover_stmt(); + recovered = true; e }); match field { @@ -6929,36 +6142,30 @@ impl<'a> Parser<'a> { let token_str = self.this_token_descr(); let mut err = self.fatal(&format!( "expected `where`, or `{{` after struct name, found {}", token_str)); - err.span_label(self.span, "expected `where`, or `{` after struct name"); + err.span_label(self.token.span, "expected `where`, or `{` after struct name"); return Err(err); } - Ok(fields) + Ok((fields, recovered)) } fn parse_tuple_struct_body(&mut self) -> PResult<'a, Vec<StructField>> { // This is the case where we find `struct Foo<T>(T) where T: Copy;` // Unit like structs are handled in parse_item_struct function - let fields = self.parse_unspanned_seq( - &token::OpenDelim(token::Paren), - &token::CloseDelim(token::Paren), - SeqSep::trailing_allowed(token::Comma), - |p| { - let attrs = p.parse_outer_attributes()?; - let lo = p.span; - let vis = p.parse_visibility(true)?; - let ty = p.parse_ty()?; - Ok(StructField { - span: lo.to(ty.span), - vis, - ident: None, - id: ast::DUMMY_NODE_ID, - ty, - attrs, - }) - })?; - - Ok(fields) + self.parse_paren_comma_seq(|p| { + let attrs = p.parse_outer_attributes()?; + let lo = p.token.span; + let vis = p.parse_visibility(true)?; + let ty = p.parse_ty()?; + Ok(StructField { + span: lo.to(ty.span), + vis, + ident: None, + id: ast::DUMMY_NODE_ID, + ty, + attrs, + }) + }).map(|(r, _)| r) } /// Parses a structure field declaration. @@ -6972,14 +6179,14 @@ impl<'a> Parser<'a> { if self.token == token::Comma { seen_comma = true; } - match self.token { + match self.token.kind { token::Comma => { self.bump(); } token::CloseDelim(token::Brace) => {} token::DocComment(_) => { let previous_span = self.prev_span; - let mut err = self.span_fatal_err(self.span, Error::UselessDocComment); + let mut err = self.span_fatal_err(self.token.span, Error::UselessDocComment); self.bump(); // consume the doc comment let comma_after_doc_seen = self.eat(&token::Comma); // `seen_comma` is always false, because we are inside doc block @@ -7026,7 +6233,7 @@ impl<'a> Parser<'a> { /// Parses an element of a struct declaration. fn parse_struct_decl_field(&mut self) -> PResult<'a, StructField> { let attrs = self.parse_outer_attributes()?; - let lo = self.span; + let lo = self.token.span; let vis = self.parse_visibility(false)?; self.parse_single_struct_field(lo, vis, attrs) } @@ -7039,17 +6246,17 @@ impl<'a> Parser<'a> { pub fn parse_visibility(&mut self, can_take_tuple: bool) -> PResult<'a, Visibility> { maybe_whole!(self, NtVis, |x| x); - self.expected_tokens.push(TokenType::Keyword(keywords::Crate)); + self.expected_tokens.push(TokenType::Keyword(kw::Crate)); if self.is_crate_vis() { self.bump(); // `crate` return Ok(respan(self.prev_span, VisibilityKind::Crate(CrateSugar::JustCrate))); } - if !self.eat_keyword(keywords::Pub) { + if !self.eat_keyword(kw::Pub) { // We need a span for our `Spanned<VisibilityKind>`, but there's inherently no // keyword to grab a span from for inherited visibility; an empty span at the // beginning of the current token would seem to be the "Schelling span". - return Ok(respan(self.span.shrink_to_lo(), VisibilityKind::Inherited)) + return Ok(respan(self.token.span.shrink_to_lo(), VisibilityKind::Inherited)) } let lo = self.prev_span; @@ -7058,7 +6265,9 @@ impl<'a> Parser<'a> { // `()` or a tuple might be allowed. For example, `struct Struct(pub (), pub (usize));`. // Because of this, we only `bump` the `(` if we're assured it is appropriate to do so // by the following tokens. - if self.look_ahead(1, |t| t.is_keyword(keywords::Crate)) { + if self.is_keyword_ahead(1, &[kw::Crate]) && + self.look_ahead(2, |t| t != &token::ModSep) // account for `pub(crate::foo)` + { // `pub(crate)` self.bump(); // `(` self.bump(); // `crate` @@ -7068,7 +6277,7 @@ impl<'a> Parser<'a> { VisibilityKind::Crate(CrateSugar::PubCrate), ); return Ok(vis) - } else if self.look_ahead(1, |t| t.is_keyword(keywords::In)) { + } else if self.is_keyword_ahead(1, &[kw::In]) { // `pub(in path)` self.bump(); // `(` self.bump(); // `in` @@ -7080,8 +6289,7 @@ impl<'a> Parser<'a> { }); return Ok(vis) } else if self.look_ahead(2, |t| t == &token::CloseDelim(token::Paren)) && - self.look_ahead(1, |t| t.is_keyword(keywords::Super) || - t.is_keyword(keywords::SelfLower)) + self.is_keyword_ahead(1, &[kw::Super, kw::SelfLower]) { // `pub(self)` or `pub(super)` self.bump(); // `(` @@ -7101,15 +6309,18 @@ impl<'a> Parser<'a> { `pub(super)`: visible only in the current module's parent `pub(in path::to::module)`: visible only on the specified path"##; let path = self.parse_path(PathStyle::Mod)?; - let sp = self.prev_span; + let sp = path.span; let help_msg = format!("make this visible only to module `{}` with `in`", path); self.expect(&token::CloseDelim(token::Paren))?; // `)` - let mut err = struct_span_err!(self.sess.span_diagnostic, sp, E0704, "{}", msg); - err.help(suggestion); - err.span_suggestion( - sp, &help_msg, format!("in {}", path), Applicability::MachineApplicable - ); - err.emit(); // emit diagnostic, but continue with public visibility + struct_span_err!(self.sess.span_diagnostic, sp, E0704, "{}", msg) + .help(suggestion) + .span_suggestion( + sp, + &help_msg, + format!("in {}", path), + Applicability::MachineApplicable, + ) + .emit(); // emit diagnostic, but continue with public visibility } } @@ -7119,14 +6330,17 @@ impl<'a> Parser<'a> { /// Parses defaultness (i.e., `default` or nothing). fn parse_defaultness(&mut self) -> Defaultness { // `pub` is included for better error messages - if self.check_keyword(keywords::Default) && - self.look_ahead(1, |t| t.is_keyword(keywords::Impl) || - t.is_keyword(keywords::Const) || - t.is_keyword(keywords::Fn) || - t.is_keyword(keywords::Unsafe) || - t.is_keyword(keywords::Extern) || - t.is_keyword(keywords::Type) || - t.is_keyword(keywords::Pub)) { + if self.check_keyword(kw::Default) && + self.is_keyword_ahead(1, &[ + kw::Impl, + kw::Const, + kw::Fn, + kw::Unsafe, + kw::Extern, + kw::Type, + kw::Pub, + ]) + { self.bump(); // `default` Defaultness::Default } else { @@ -7134,39 +6348,8 @@ impl<'a> Parser<'a> { } } - fn maybe_consume_incorrect_semicolon(&mut self, items: &[P<Item>]) -> bool { - if self.eat(&token::Semi) { - let mut err = self.struct_span_err(self.prev_span, "expected item, found `;`"); - err.span_suggestion_short( - self.prev_span, - "remove this semicolon", - String::new(), - Applicability::MachineApplicable, - ); - if !items.is_empty() { - let previous_item = &items[items.len()-1]; - let previous_item_kind_name = match previous_item.node { - // say "braced struct" because tuple-structs and - // braceless-empty-struct declarations do take a semicolon - ItemKind::Struct(..) => Some("braced struct"), - ItemKind::Enum(..) => Some("enum"), - ItemKind::Trait(..) => Some("trait"), - ItemKind::Union(..) => Some("union"), - _ => None, - }; - if let Some(name) = previous_item_kind_name { - err.help(&format!("{} declarations are not followed by a semicolon", name)); - } - } - err.emit(); - true - } else { - false - } - } - /// Given a termination token, parses all of the items in a module. - fn parse_mod_items(&mut self, term: &token::Token, inner_lo: Span) -> PResult<'a, Mod> { + fn parse_mod_items(&mut self, term: &TokenKind, inner_lo: Span) -> PResult<'a, Mod> { let mut items = vec![]; while let Some(item) = self.parse_item()? { items.push(item); @@ -7177,12 +6360,12 @@ impl<'a> Parser<'a> { let token_str = self.this_token_descr(); if !self.maybe_consume_incorrect_semicolon(&items) { let mut err = self.fatal(&format!("expected item, found {}", token_str)); - err.span_label(self.span, "expected item"); + err.span_label(self.token.span, "expected item"); return Err(err); } } - let hi = if self.span.is_dummy() { + let hi = if self.token.span.is_dummy() { inner_lo } else { self.prev_span @@ -7221,7 +6404,7 @@ impl<'a> Parser<'a> { (!self.cfg_mods || strip_unconfigured.in_cfg(&outer_attrs), outer_attrs) }; - let id_span = self.span; + let id_span = self.token.span; let id = self.parse_ident()?; if self.eat(&token::Semi) { if in_cfg && self.recurse_into_file_modules { @@ -7232,21 +6415,15 @@ impl<'a> Parser<'a> { self.eval_src_mod(path, directory_ownership, id.to_string(), id_span)?; // Record that we fetched the mod from an external file if warn { - let attr = Attribute { - id: attr::mk_attr_id(), - style: ast::AttrStyle::Outer, - path: ast::Path::from_ident(Ident::from_str("warn_directory_ownership")), - tokens: TokenStream::empty(), - is_sugared_doc: false, - span: syntax_pos::DUMMY_SP, - }; + let attr = attr::mk_attr_outer( + attr::mk_word_item(Ident::with_empty_ctxt(sym::warn_directory_ownership))); attr::mark_known(&attr); attrs.push(attr); } Ok((id, ItemKind::Mod(module), Some(attrs))) } else { let placeholder = ast::Mod { - inner: syntax_pos::DUMMY_SP, + inner: DUMMY_SP, items: Vec::new(), inline: false }; @@ -7257,7 +6434,7 @@ impl<'a> Parser<'a> { self.push_directory(id, &outer_attrs); self.expect(&token::OpenDelim(token::Brace))?; - let mod_inner_lo = self.span; + let mod_inner_lo = self.token.span; let attrs = self.parse_inner_attributes()?; let module = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?; @@ -7267,7 +6444,7 @@ impl<'a> Parser<'a> { } fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) { - if let Some(path) = attr::first_attr_value_str_by_name(attrs, "path") { + if let Some(path) = attr::first_attr_value_str_by_name(attrs, sym::path) { self.directory.path.to_mut().push(&path.as_str()); self.directory.ownership = DirectoryOwnership::Owned { relative: None }; } else { @@ -7287,7 +6464,7 @@ impl<'a> Parser<'a> { } pub fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> { - if let Some(s) = attr::first_attr_value_str_by_name(attrs, "path") { + if let Some(s) = attr::first_attr_value_str_by_name(attrs, sym::path) { let s = s.as_str(); // On windows, the base path might have the form @@ -7446,12 +6623,13 @@ impl<'a> Parser<'a> { } /// Reads a module from a source file. - fn eval_src_mod(&mut self, - path: PathBuf, - directory_ownership: DirectoryOwnership, - name: String, - id_sp: Span) - -> PResult<'a, (ast::Mod, Vec<Attribute> )> { + fn eval_src_mod( + &mut self, + path: PathBuf, + directory_ownership: DirectoryOwnership, + name: String, + id_sp: Span, + ) -> PResult<'a, (ast::Mod, Vec<Attribute>)> { let mut included_mod_stack = self.sess.included_mod_stack.borrow_mut(); if let Some(i) = included_mod_stack.iter().position(|p| *p == path) { let mut err = String::from("circular modules: "); @@ -7469,7 +6647,7 @@ impl<'a> Parser<'a> { let mut p0 = new_sub_parser_from_file(self.sess, &path, directory_ownership, Some(name), id_sp); p0.cfg_mods = self.cfg_mods; - let mod_inner_lo = p0.span; + let mod_inner_lo = p0.token.span; let mod_attrs = p0.parse_inner_attributes()?; let mut m0 = p0.parse_mod_items(&token::Eof, mod_inner_lo)?; m0.inline = false; @@ -7478,15 +6656,20 @@ impl<'a> Parser<'a> { } /// Parses a function declaration from a foreign module. - fn parse_item_foreign_fn(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) - -> PResult<'a, ForeignItem> { - self.expect_keyword(keywords::Fn)?; + fn parse_item_foreign_fn( + &mut self, + vis: ast::Visibility, + lo: Span, + attrs: Vec<Attribute>, + extern_sp: Span, + ) -> PResult<'a, ForeignItem> { + self.expect_keyword(kw::Fn)?; let (ident, mut generics) = self.parse_fn_header()?; let decl = self.parse_fn_decl(true)?; generics.where_clause = self.parse_where_clause()?; - let hi = self.span; - self.expect(&token::Semi)?; + let hi = self.token.span; + self.parse_semi_or_incorrect_foreign_fn_body(&ident, extern_sp)?; Ok(ast::ForeignItem { ident, attrs, @@ -7501,11 +6684,11 @@ impl<'a> Parser<'a> { /// Assumes that the `static` keyword is already parsed. fn parse_item_foreign_static(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, ForeignItem> { - let mutbl = self.eat_keyword(keywords::Mut); + let mutbl = self.parse_mutability(); let ident = self.parse_ident()?; self.expect(&token::Colon)?; let ty = self.parse_ty()?; - let hi = self.span; + let hi = self.token.span; self.expect(&token::Semi)?; Ok(ForeignItem { ident, @@ -7520,18 +6703,18 @@ impl<'a> Parser<'a> { /// Parses a type from a foreign module. fn parse_item_foreign_type(&mut self, vis: ast::Visibility, lo: Span, attrs: Vec<Attribute>) -> PResult<'a, ForeignItem> { - self.expect_keyword(keywords::Type)?; + self.expect_keyword(kw::Type)?; let ident = self.parse_ident()?; - let hi = self.span; + let hi = self.token.span; self.expect(&token::Semi)?; Ok(ast::ForeignItem { - ident: ident, - attrs: attrs, + ident, + attrs, node: ForeignItemKind::Ty, id: ast::DUMMY_NODE_ID, span: lo.to(hi), - vis: vis + vis }) } @@ -7539,7 +6722,7 @@ impl<'a> Parser<'a> { let error_msg = "crate name using dashes are not valid in `extern crate` statements"; let suggestion_msg = "if the original crate name uses dashes you need to use underscores \ in the code"; - let mut ident = if self.token.is_keyword(keywords::SelfLower) { + let mut ident = if self.token.is_keyword(kw::SelfLower) { self.parse_path_segment_ident() } else { self.parse_ident() @@ -7548,7 +6731,7 @@ impl<'a> Parser<'a> { let mut replacement = vec![]; let mut fixed_crate_name = false; // Accept `extern crate name-like-this` for better diagnostics - let dash = token::Token::BinOp(token::BinOpToken::Minus); + let dash = token::BinOp(token::BinOpToken::Minus); if self.token == dash { // Do not include `-` as part of the expected tokens list while self.eat(&dash) { fixed_crate_name = true; @@ -7564,14 +6747,10 @@ impl<'a> Parser<'a> { } ident = Ident::from_str(&fixed_name).with_span_pos(fixed_name_sp); - let mut err = self.struct_span_err(fixed_name_sp, error_msg); - err.span_label(fixed_name_sp, "dash-separated idents are not valid"); - err.multipart_suggestion( - suggestion_msg, - replacement, - Applicability::MachineApplicable, - ); - err.emit(); + self.struct_span_err(fixed_name_sp, error_msg) + .span_label(fixed_name_sp, "dash-separated idents are not valid") + .multipart_suggestion(suggestion_msg, replacement, Applicability::MachineApplicable) + .emit(); } Ok(ident) } @@ -7613,12 +6792,14 @@ impl<'a> Parser<'a> { /// extern "C" {} /// extern {} /// ``` - fn parse_item_foreign_mod(&mut self, - lo: Span, - opt_abi: Option<Abi>, - visibility: Visibility, - mut attrs: Vec<Attribute>) - -> PResult<'a, P<Item>> { + fn parse_item_foreign_mod( + &mut self, + lo: Span, + opt_abi: Option<Abi>, + visibility: Visibility, + mut attrs: Vec<Attribute>, + extern_sp: Span, + ) -> PResult<'a, P<Item>> { self.expect(&token::OpenDelim(token::Brace))?; let abi = opt_abi.unwrap_or(Abi::C); @@ -7627,7 +6808,7 @@ impl<'a> Parser<'a> { let mut foreign_items = vec![]; while !self.eat(&token::CloseDelim(token::Brace)) { - foreign_items.push(self.parse_foreign_item()?); + foreign_items.push(self.parse_foreign_item(extern_sp)?); } let prev_span = self.prev_span; @@ -7635,44 +6816,33 @@ impl<'a> Parser<'a> { abi, items: foreign_items }; - let invalid = keywords::Invalid.ident(); + let invalid = Ident::invalid(); Ok(self.mk_item(lo.to(prev_span), invalid, ItemKind::ForeignMod(m), visibility, attrs)) } - /// Parses `type Foo = Bar;` - /// or - /// `existential type Foo: Bar;` - /// or - /// `return `None`` + /// Parses `type Foo = Bar;` or returns `None` /// without modifying the parser state. fn eat_type(&mut self) -> Option<PResult<'a, (Ident, AliasKind, ast::Generics)>> { // This parses the grammar: // Ident ["<"...">"] ["where" ...] ("=" | ":") Ty ";" - if self.check_keyword(keywords::Type) || - self.check_keyword(keywords::Existential) && - self.look_ahead(1, |t| t.is_keyword(keywords::Type)) { - let existential = self.eat_keyword(keywords::Existential); - assert!(self.eat_keyword(keywords::Type)); - Some(self.parse_existential_or_alias(existential)) + if self.eat_keyword(kw::Type) { + Some(self.parse_type_alias()) } else { None } } - /// Parses a type alias or existential type. - fn parse_existential_or_alias( - &mut self, - existential: bool, - ) -> PResult<'a, (Ident, AliasKind, ast::Generics)> { + /// Parses a type alias or opaque type. + fn parse_type_alias(&mut self) -> PResult<'a, (Ident, AliasKind, ast::Generics)> { let ident = self.parse_ident()?; let mut tps = self.parse_generics()?; tps.where_clause = self.parse_where_clause()?; - let alias = if existential { - self.expect(&token::Colon)?; + self.expect(&token::Eq)?; + let alias = if self.check_keyword(kw::Impl) { + self.bump(); let bounds = self.parse_generic_bounds(Some(self.prev_span))?; - AliasKind::Existential(bounds) + AliasKind::OpaqueTy(bounds) } else { - self.expect(&token::Eq)?; let ty = self.parse_ty()?; AliasKind::Weak(ty) }; @@ -7683,58 +6853,61 @@ impl<'a> Parser<'a> { /// Parses the part of an enum declaration following the `{`. fn parse_enum_def(&mut self, _generics: &ast::Generics) -> PResult<'a, EnumDef> { let mut variants = Vec::new(); - let mut all_nullary = true; - let mut any_disr = vec![]; while self.token != token::CloseDelim(token::Brace) { let variant_attrs = self.parse_outer_attributes()?; - let vlo = self.span; + let vlo = self.token.span; - let struct_def; - let mut disr_expr = None; + self.eat_bad_pub(); let ident = self.parse_ident()?; - if self.check(&token::OpenDelim(token::Brace)) { + + let struct_def = if self.check(&token::OpenDelim(token::Brace)) { // Parse a struct variant. - all_nullary = false; - struct_def = VariantData::Struct(self.parse_record_struct_body()?, - ast::DUMMY_NODE_ID); + let (fields, recovered) = self.parse_record_struct_body()?; + VariantData::Struct(fields, recovered) } else if self.check(&token::OpenDelim(token::Paren)) { - all_nullary = false; - struct_def = VariantData::Tuple(self.parse_tuple_struct_body()?, - ast::DUMMY_NODE_ID); - } else if self.eat(&token::Eq) { - disr_expr = Some(AnonConst { + VariantData::Tuple( + self.parse_tuple_struct_body()?, + ast::DUMMY_NODE_ID, + ) + } else { + VariantData::Unit(ast::DUMMY_NODE_ID) + }; + + let disr_expr = if self.eat(&token::Eq) { + Some(AnonConst { id: ast::DUMMY_NODE_ID, value: self.parse_expr()?, - }); - if let Some(sp) = disr_expr.as_ref().map(|c| c.value.span) { - any_disr.push(sp); - } - struct_def = VariantData::Unit(ast::DUMMY_NODE_ID); + }) } else { - struct_def = VariantData::Unit(ast::DUMMY_NODE_ID); - } + None + }; let vr = ast::Variant_ { ident, + id: ast::DUMMY_NODE_ID, attrs: variant_attrs, data: struct_def, disr_expr, }; variants.push(respan(vlo.to(self.prev_span), vr)); - if !self.eat(&token::Comma) { break; } - } - self.expect(&token::CloseDelim(token::Brace))?; - if !any_disr.is_empty() && !all_nullary { - let mut err =self.struct_span_err( - any_disr.clone(), - "discriminator values can only be used with a field-less enum", - ); - for sp in any_disr { - err.span_label(sp, "only valid in field-less enums"); + if !self.eat(&token::Comma) { + if self.token.is_ident() && !self.token.is_reserved_ident() { + let sp = self.sess.source_map().next_point(self.prev_span); + self.struct_span_err(sp, "missing comma") + .span_suggestion_short( + sp, + "missing comma", + ",".to_owned(), + Applicability::MaybeIncorrect, + ) + .emit(); + } else { + break; + } } - err.emit(); } + self.expect(&token::CloseDelim(token::Brace))?; Ok(ast::EnumDef { variants }) } @@ -7757,24 +6930,26 @@ impl<'a> Parser<'a> { /// Parses a string as an ABI spec on an extern type or module. Consumes /// the `extern` keyword, if one is found. fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> { - match self.token { - token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => { - let sp = self.span; - self.expect_no_suffix(sp, "ABI spec", suf); + match self.token.kind { + token::Literal(token::Lit { kind: token::Str, symbol, suffix }) | + token::Literal(token::Lit { kind: token::StrRaw(..), symbol, suffix }) => { + let sp = self.token.span; + self.expect_no_suffix(sp, "an ABI spec", suffix); self.bump(); - match abi::lookup(&s.as_str()) { + match abi::lookup(&symbol.as_str()) { Some(abi) => Ok(Some(abi)), None => { let prev_span = self.prev_span; - let mut err = struct_span_err!( + struct_span_err!( self.sess.span_diagnostic, prev_span, E0703, "invalid ABI: found `{}`", - s); - err.span_label(prev_span, "invalid ABI"); - err.help(&format!("valid ABIs: {}", abi::all_names().join(", "))); - err.emit(); + symbol + ) + .span_label(prev_span, "invalid ABI") + .help(&format!("valid ABIs: {}", abi::all_names().join(", "))) + .emit(); Ok(None) } } @@ -7785,13 +6960,13 @@ impl<'a> Parser<'a> { } fn is_static_global(&mut self) -> bool { - if self.check_keyword(keywords::Static) { + if self.check_keyword(kw::Static) { // Check if this could be a closure !self.look_ahead(1, |token| { - if token.is_keyword(keywords::Move) { + if token.is_keyword(kw::Move) { return true; } - match *token { + match token.kind { token::BinOp(token::Or) | token::OrOr => true, _ => false, } @@ -7854,28 +7029,30 @@ impl<'a> Parser<'a> { Some(P(item)) }); - let lo = self.span; + let lo = self.token.span; let visibility = self.parse_visibility(false)?; - if self.eat_keyword(keywords::Use) { + if self.eat_keyword(kw::Use) { // USE ITEM let item_ = ItemKind::Use(P(self.parse_use_tree()?)); self.expect(&token::Semi)?; let span = lo.to(self.prev_span); - let item = self.mk_item(span, keywords::Invalid.ident(), item_, visibility, attrs); + let item = + self.mk_item(span, Ident::invalid(), item_, visibility, attrs); return Ok(Some(item)); } - if self.eat_keyword(keywords::Extern) { - if self.eat_keyword(keywords::Crate) { + if self.eat_keyword(kw::Extern) { + let extern_sp = self.prev_span; + if self.eat_keyword(kw::Crate) { return Ok(Some(self.parse_item_extern_crate(lo, visibility, attrs)?)); } let opt_abi = self.parse_opt_abi()?; - if self.eat_keyword(keywords::Fn) { + if self.eat_keyword(kw::Fn) { // EXTERN FUNCTION ITEM let fn_span = self.prev_span; let abi = opt_abi.unwrap_or(Abi::C); @@ -7892,7 +7069,9 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } else if self.check(&token::OpenDelim(token::Brace)) { - return Ok(Some(self.parse_item_foreign_mod(lo, opt_abi, visibility, attrs)?)); + return Ok(Some( + self.parse_item_foreign_mod(lo, opt_abi, visibility, attrs, extern_sp)?, + )); } self.unexpected()?; @@ -7901,7 +7080,7 @@ impl<'a> Parser<'a> { if self.is_static_global() { self.bump(); // STATIC ITEM - let m = if self.eat_keyword(keywords::Mut) { + let m = if self.eat_keyword(kw::Mut) { Mutability::Mutable } else { Mutability::Immutable @@ -7915,11 +7094,11 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.eat_keyword(keywords::Const) { + if self.eat_keyword(kw::Const) { let const_span = self.prev_span; - if self.check_keyword(keywords::Fn) - || (self.check_keyword(keywords::Unsafe) - && self.look_ahead(1, |t| t.is_keyword(keywords::Fn))) { + if self.check_keyword(kw::Fn) + || (self.check_keyword(kw::Unsafe) + && self.is_keyword_ahead(1, &[kw::Fn])) { // CONST FUNCTION ITEM let unsafety = self.parse_unsafety(); self.bump(); @@ -7938,18 +7117,17 @@ impl<'a> Parser<'a> { } // CONST ITEM - if self.eat_keyword(keywords::Mut) { + if self.eat_keyword(kw::Mut) { let prev_span = self.prev_span; - let mut err = self.diagnostic() - .struct_span_err(prev_span, "const globals cannot be mutable"); - err.span_label(prev_span, "cannot be mutable"); - err.span_suggestion( - const_span, - "you might want to declare a static instead", - "static".to_owned(), - Applicability::MaybeIncorrect, - ); - err.emit(); + self.struct_span_err(prev_span, "const globals cannot be mutable") + .span_label(prev_span, "cannot be mutable") + .span_suggestion( + const_span, + "you might want to declare a static instead", + "static".to_owned(), + Applicability::MaybeIncorrect, + ) + .emit(); } let (ident, item_, extra_attrs) = self.parse_item_const(None)?; let prev_span = self.prev_span; @@ -7961,55 +7139,45 @@ impl<'a> Parser<'a> { return Ok(Some(item)); } - // `unsafe async fn` or `async fn` - if ( - self.check_keyword(keywords::Unsafe) && - self.look_ahead(1, |t| t.is_keyword(keywords::Async)) - ) || ( - self.check_keyword(keywords::Async) && - self.look_ahead(1, |t| t.is_keyword(keywords::Fn)) - ) - { - // ASYNC FUNCTION ITEM - let unsafety = self.parse_unsafety(); - self.expect_keyword(keywords::Async)?; - let async_span = self.prev_span; - self.expect_keyword(keywords::Fn)?; - let fn_span = self.prev_span; - let (ident, item_, extra_attrs) = - self.parse_item_fn(unsafety, - respan(async_span, IsAsync::Async { - closure_id: ast::DUMMY_NODE_ID, - return_impl_trait_id: ast::DUMMY_NODE_ID, - }), - respan(fn_span, Constness::NotConst), - Abi::Rust)?; - let prev_span = self.prev_span; - let item = self.mk_item(lo.to(prev_span), - ident, - item_, - visibility, - maybe_append(attrs, extra_attrs)); - if self.span.rust_2015() { - self.diagnostic().struct_span_err_with_code( - async_span, - "`async fn` is not permitted in the 2015 edition", - DiagnosticId::Error("E0670".into()) - ).emit(); + // Parse `async unsafe? fn`. + if self.check_keyword(kw::Async) { + let async_span = self.token.span; + if self.is_keyword_ahead(1, &[kw::Fn]) + || self.is_keyword_ahead(2, &[kw::Fn]) + { + // ASYNC FUNCTION ITEM + self.bump(); // `async` + let unsafety = self.parse_unsafety(); // `unsafe`? + self.expect_keyword(kw::Fn)?; // `fn` + let fn_span = self.prev_span; + let (ident, item_, extra_attrs) = + self.parse_item_fn(unsafety, + respan(async_span, IsAsync::Async { + closure_id: ast::DUMMY_NODE_ID, + return_impl_trait_id: ast::DUMMY_NODE_ID, + }), + respan(fn_span, Constness::NotConst), + Abi::Rust)?; + let prev_span = self.prev_span; + let item = self.mk_item(lo.to(prev_span), + ident, + item_, + visibility, + maybe_append(attrs, extra_attrs)); + self.ban_async_in_2015(async_span); + return Ok(Some(item)); } - return Ok(Some(item)); } - if self.check_keyword(keywords::Unsafe) && - (self.look_ahead(1, |t| t.is_keyword(keywords::Trait)) || - self.look_ahead(1, |t| t.is_keyword(keywords::Auto))) + if self.check_keyword(kw::Unsafe) && + self.is_keyword_ahead(1, &[kw::Trait, kw::Auto]) { // UNSAFE TRAIT ITEM self.bump(); // `unsafe` - let is_auto = if self.eat_keyword(keywords::Trait) { + let is_auto = if self.eat_keyword(kw::Trait) { IsAuto::No } else { - self.expect_keyword(keywords::Auto)?; - self.expect_keyword(keywords::Trait)?; + self.expect_keyword(kw::Auto)?; + self.expect_keyword(kw::Trait)?; IsAuto::Yes }; let (ident, item_, extra_attrs) = @@ -8022,23 +7190,21 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.check_keyword(keywords::Impl) || - self.check_keyword(keywords::Unsafe) && - self.look_ahead(1, |t| t.is_keyword(keywords::Impl)) || - self.check_keyword(keywords::Default) && - self.look_ahead(1, |t| t.is_keyword(keywords::Impl)) || - self.check_keyword(keywords::Default) && - self.look_ahead(1, |t| t.is_keyword(keywords::Unsafe)) { + if self.check_keyword(kw::Impl) || + self.check_keyword(kw::Unsafe) && + self.is_keyword_ahead(1, &[kw::Impl]) || + self.check_keyword(kw::Default) && + self.is_keyword_ahead(1, &[kw::Impl, kw::Unsafe]) { // IMPL ITEM let defaultness = self.parse_defaultness(); let unsafety = self.parse_unsafety(); - self.expect_keyword(keywords::Impl)?; + self.expect_keyword(kw::Impl)?; let (ident, item, extra_attrs) = self.parse_item_impl(unsafety, defaultness)?; let span = lo.to(self.prev_span); return Ok(Some(self.mk_item(span, ident, item, visibility, maybe_append(attrs, extra_attrs)))); } - if self.check_keyword(keywords::Fn) { + if self.check_keyword(kw::Fn) { // FUNCTION ITEM self.bump(); let fn_span = self.prev_span; @@ -8055,18 +7221,18 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.check_keyword(keywords::Unsafe) + if self.check_keyword(kw::Unsafe) && self.look_ahead(1, |t| *t != token::OpenDelim(token::Brace)) { // UNSAFE FUNCTION ITEM self.bump(); // `unsafe` // `{` is also expected after `unsafe`, in case of error, include it in the diagnostic self.check(&token::OpenDelim(token::Brace)); - let abi = if self.eat_keyword(keywords::Extern) { + let abi = if self.eat_keyword(kw::Extern) { self.parse_opt_abi()?.unwrap_or(Abi::C) } else { Abi::Rust }; - self.expect_keyword(keywords::Fn)?; + self.expect_keyword(kw::Fn)?; let fn_span = self.prev_span; let (ident, item_, extra_attrs) = self.parse_item_fn(Unsafety::Unsafe, @@ -8081,7 +7247,7 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.eat_keyword(keywords::Mod) { + if self.eat_keyword(kw::Mod) { // MODULE ITEM let (ident, item_, extra_attrs) = self.parse_item_mod(&attrs[..])?; @@ -8097,8 +7263,8 @@ impl<'a> Parser<'a> { let (ident, alias, generics) = type_?; // TYPE ITEM let item_ = match alias { - AliasKind::Weak(ty) => ItemKind::Ty(ty, generics), - AliasKind::Existential(bounds) => ItemKind::Existential(bounds, generics), + AliasKind::Weak(ty) => ItemKind::TyAlias(ty, generics), + AliasKind::OpaqueTy(bounds) => ItemKind::OpaqueTy(bounds, generics), }; let prev_span = self.prev_span; let item = self.mk_item(lo.to(prev_span), @@ -8108,7 +7274,7 @@ impl<'a> Parser<'a> { attrs); return Ok(Some(item)); } - if self.eat_keyword(keywords::Enum) { + if self.eat_keyword(kw::Enum) { // ENUM ITEM let (ident, item_, extra_attrs) = self.parse_item_enum()?; let prev_span = self.prev_span; @@ -8119,15 +7285,15 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.check_keyword(keywords::Trait) - || (self.check_keyword(keywords::Auto) - && self.look_ahead(1, |t| t.is_keyword(keywords::Trait))) + if self.check_keyword(kw::Trait) + || (self.check_keyword(kw::Auto) + && self.is_keyword_ahead(1, &[kw::Trait])) { - let is_auto = if self.eat_keyword(keywords::Trait) { + let is_auto = if self.eat_keyword(kw::Trait) { IsAuto::No } else { - self.expect_keyword(keywords::Auto)?; - self.expect_keyword(keywords::Trait)?; + self.expect_keyword(kw::Auto)?; + self.expect_keyword(kw::Trait)?; IsAuto::Yes }; // TRAIT ITEM @@ -8141,7 +7307,7 @@ impl<'a> Parser<'a> { maybe_append(attrs, extra_attrs)); return Ok(Some(item)); } - if self.eat_keyword(keywords::Struct) { + if self.eat_keyword(kw::Struct) { // STRUCT ITEM let (ident, item_, extra_attrs) = self.parse_item_struct()?; let prev_span = self.prev_span; @@ -8178,9 +7344,9 @@ impl<'a> Parser<'a> { // // pub S {} // ^^^ `sp` points here - let sp = self.prev_span.between(self.span); - let full_sp = self.prev_span.to(self.span); - let ident_sp = self.span; + let sp = self.prev_span.between(self.token.span); + let full_sp = self.prev_span.to(self.token.span); + let ident_sp = self.token.span; if self.look_ahead(1, |t| *t == token::OpenDelim(token::Brace)) { // possible public struct definition where `struct` was forgotten let ident = self.parse_ident().unwrap(); @@ -8195,7 +7361,9 @@ impl<'a> Parser<'a> { } else if self.look_ahead(1, |t| *t == token::OpenDelim(token::Paren)) { let ident = self.parse_ident().unwrap(); self.bump(); // `(` - let kw_name = if let Ok(Some(_)) = self.parse_self_arg() { + let kw_name = if let Ok(Some(_)) = self.parse_self_arg_with_attrs() + .map_err(|mut e| e.cancel()) + { "method" } else { "function" @@ -8214,11 +7382,11 @@ impl<'a> Parser<'a> { } else { ("fn` or `struct", "function or struct", true) }; - self.consume_block(token::Brace); let msg = format!("missing `{}` for {} definition", kw, kw_name); let mut err = self.diagnostic().struct_span_err(sp, &msg); if !ambiguous { + self.consume_block(token::Brace); let suggestion = format!("add `{}` here to parse `{}` as a public {}", kw, ident, @@ -8227,7 +7395,7 @@ impl<'a> Parser<'a> { sp, &suggestion, format!(" {} ", kw), Applicability::MachineApplicable ); } else { - if let Ok(snippet) = self.sess.source_map().span_to_snippet(ident_sp) { + if let Ok(snippet) = self.span_to_snippet(ident_sp) { err.span_suggestion( full_sp, "if you meant to call a macro, try", @@ -8246,7 +7414,9 @@ impl<'a> Parser<'a> { self.eat_to_tokens(&[&token::Gt]); self.bump(); // `>` let (kw, kw_name, ambiguous) = if self.eat(&token::OpenDelim(token::Paren)) { - if let Ok(Some(_)) = self.parse_self_arg() { + if let Ok(Some(_)) = self.parse_self_arg_with_attrs() + .map_err(|mut e| e.cancel()) + { ("fn", "method", false) } else { ("fn", "function", false) @@ -8272,22 +7442,35 @@ impl<'a> Parser<'a> { self.parse_macro_use_or_failure(attrs, macros_allowed, attributes_allowed, lo, visibility) } + /// We are parsing `async fn`. If we are on Rust 2015, emit an error. + fn ban_async_in_2015(&self, async_span: Span) { + if async_span.rust_2015() { + self.diagnostic() + .struct_span_err_with_code( + async_span, + "`async fn` is not permitted in the 2015 edition", + DiagnosticId::Error("E0670".into()) + ) + .emit(); + } + } + /// Parses a foreign item. - crate fn parse_foreign_item(&mut self) -> PResult<'a, ForeignItem> { + crate fn parse_foreign_item(&mut self, extern_sp: Span) -> PResult<'a, ForeignItem> { maybe_whole!(self, NtForeignItem, |ni| ni); let attrs = self.parse_outer_attributes()?; - let lo = self.span; + let lo = self.token.span; let visibility = self.parse_visibility(false)?; // FOREIGN STATIC ITEM // Treat `const` as `static` for error recovery, but don't add it to expected tokens. - if self.check_keyword(keywords::Static) || self.token.is_keyword(keywords::Const) { - if self.token.is_keyword(keywords::Const) { + if self.check_keyword(kw::Static) || self.token.is_keyword(kw::Const) { + if self.token.is_keyword(kw::Const) { self.diagnostic() - .struct_span_err(self.span, "extern items cannot be `const`") + .struct_span_err(self.token.span, "extern items cannot be `const`") .span_suggestion( - self.span, + self.token.span, "try using a static value", "static".to_owned(), Applicability::MachineApplicable @@ -8297,11 +7480,11 @@ impl<'a> Parser<'a> { return Ok(self.parse_item_foreign_static(visibility, lo, attrs)?); } // FOREIGN FUNCTION ITEM - if self.check_keyword(keywords::Fn) { - return Ok(self.parse_item_foreign_fn(visibility, lo, attrs)?); + if self.check_keyword(kw::Fn) { + return Ok(self.parse_item_foreign_fn(visibility, lo, attrs, extern_sp)?); } // FOREIGN TYPE ITEM - if self.check_keyword(keywords::Type) { + if self.check_keyword(kw::Type) { return Ok(self.parse_item_foreign_type(visibility, lo, attrs)?); } @@ -8309,7 +7492,7 @@ impl<'a> Parser<'a> { Some(mac) => { Ok( ForeignItem { - ident: keywords::Invalid.ident(), + ident: Ident::invalid(), span: lo.to(self.prev_span), id: ast::DUMMY_NODE_ID, attrs, @@ -8338,40 +7521,31 @@ impl<'a> Parser<'a> { visibility: Visibility ) -> PResult<'a, Option<P<Item>>> { if macros_allowed && self.token.is_path_start() && - !(self.is_async_fn() && self.span.rust_2015()) { + !(self.is_async_fn() && self.token.span.rust_2015()) { // MACRO INVOCATION ITEM let prev_span = self.prev_span; self.complain_if_pub_macro(&visibility.node, prev_span); - let mac_lo = self.span; + let mac_lo = self.token.span; // item macro. - let pth = self.parse_path(PathStyle::Mod)?; + let path = self.parse_path(PathStyle::Mod)?; self.expect(&token::Not)?; - - // a 'special' identifier (like what `macro_rules!` uses) - // is optional. We should eventually unify invoc syntax - // and remove this. - let id = if self.token.is_ident() { - self.parse_ident()? - } else { - keywords::Invalid.ident() // no special identifier - }; - // eat a matched-delimiter token tree: let (delim, tts) = self.expect_delimited_token_tree()?; - if delim != MacDelimiter::Brace { - if !self.eat(&token::Semi) { - self.span_err(self.prev_span, - "macros that expand to items must either \ - be surrounded with braces or followed by \ - a semicolon"); - } + if delim != MacDelimiter::Brace && !self.eat(&token::Semi) { + self.report_invalid_macro_expansion_item(); } let hi = self.prev_span; - let mac = respan(mac_lo.to(hi), Mac_ { path: pth, tts, delim }); - let item = self.mk_item(lo.to(hi), id, ItemKind::Mac(mac), visibility, attrs); + let mac = respan(mac_lo.to(hi), Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + }); + let item = + self.mk_item(lo.to(hi), Ident::invalid(), ItemKind::Mac(mac), visibility, attrs); return Ok(Some(item)); } @@ -8394,12 +7568,12 @@ impl<'a> Parser<'a> { at_end: &mut bool) -> PResult<'a, Option<Mac>> { if self.token.is_path_start() && - !(self.is_async_fn() && self.span.rust_2015()) { + !(self.is_async_fn() && self.token.span.rust_2015()) { let prev_span = self.prev_span; - let lo = self.span; - let pth = self.parse_path(PathStyle::Mod)?; + let lo = self.token.span; + let path = self.parse_path(PathStyle::Mod)?; - if pth.segments.len() == 1 { + if path.segments.len() == 1 { if !self.eat(&token::Not) { return Err(self.missing_assoc_item_kind_err(item_kind, prev_span)); } @@ -8419,7 +7593,12 @@ impl<'a> Parser<'a> { self.expect(&token::Semi)?; } - Ok(Some(respan(lo.to(self.prev_span), Mac_ { path: pth, tts, delim }))) + Ok(Some(respan(lo.to(self.prev_span), Mac_ { + path, + tts, + delim, + prior_type_ascription: self.last_type_ascription, + }))) } else { Ok(None) } @@ -8432,7 +7611,7 @@ impl<'a> Parser<'a> { let mut tokens = Vec::new(); let prev_collecting = match self.token_cursor.frame.last_token { LastToken::Collecting(ref mut list) => { - Some(mem::replace(list, Vec::new())) + Some(mem::take(list)) } LastToken::Was(ref mut last) => { tokens.extend(last.take()); @@ -8444,14 +7623,27 @@ impl<'a> Parser<'a> { let ret = f(self); let last_token = if self.token_cursor.stack.len() == prev { &mut self.token_cursor.frame.last_token + } else if self.token_cursor.stack.get(prev).is_none() { + // This can happen due to a bad interaction of two unrelated recovery mechanisms with + // mismatched delimiters *and* recovery lookahead on the likely typo `pub ident(` + // (#62881). + return Ok((ret?, TokenStream::new(vec![]))); } else { &mut self.token_cursor.stack[prev].last_token }; // Pull out the tokens that we've collected from the call to `f` above. let mut collected_tokens = match *last_token { - LastToken::Collecting(ref mut v) => mem::replace(v, Vec::new()), - LastToken::Was(_) => panic!("our vector went away?"), + LastToken::Collecting(ref mut v) => mem::take(v), + LastToken::Was(ref was) => { + let msg = format!("our vector went away? - found Was({:?})", was); + debug!("collect_tokens: {}", msg); + self.sess.span_diagnostic.delay_span_bug(self.token.span, &msg); + // This can happen due to a bad interaction of two unrelated recovery mechanisms + // with mismatched delimiters *and* recovery lookahead on the likely typo + // `pub ident(` (#62895, different but similar to the case above). + return Ok((ret?, TokenStream::new(vec![]))); + } }; // If we're not at EOF our current token wasn't actually consumed by @@ -8503,14 +7695,14 @@ impl<'a> Parser<'a> { /// PATH [`as` IDENT] /// ``` fn parse_use_tree(&mut self) -> PResult<'a, UseTree> { - let lo = self.span; + let lo = self.token.span; let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo() }; let kind = if self.check(&token::OpenDelim(token::Brace)) || self.check(&token::BinOp(token::Star)) || self.is_import_coupler() { // `use *;` or `use ::*;` or `use {...};` or `use ::{...};` - let mod_sep_ctxt = self.span.ctxt(); + let mod_sep_ctxt = self.token.span.ctxt(); if self.eat(&token::ModSep) { prefix.segments.push( PathSegment::path_root(lo.shrink_to_lo().with_ctxt(mod_sep_ctxt)) @@ -8546,15 +7738,12 @@ impl<'a> Parser<'a> { /// USE_TREE_LIST = Ø | (USE_TREE `,`)* USE_TREE [`,`] /// ``` fn parse_use_tree_list(&mut self) -> PResult<'a, Vec<(UseTree, ast::NodeId)>> { - self.parse_unspanned_seq(&token::OpenDelim(token::Brace), - &token::CloseDelim(token::Brace), - SeqSep::trailing_allowed(token::Comma), |this| { - Ok((this.parse_use_tree()?, ast::DUMMY_NODE_ID)) - }) + self.parse_delim_comma_seq(token::Brace, |p| Ok((p.parse_use_tree()?, ast::DUMMY_NODE_ID))) + .map(|(r, _)| r) } fn parse_rename(&mut self) -> PResult<'a, Option<Ident>> { - if self.eat_keyword(keywords::As) { + if self.eat_keyword(kw::As) { self.parse_ident_or_underscore().map(Some) } else { Ok(None) @@ -8563,19 +7752,21 @@ impl<'a> Parser<'a> { /// Parses a source module as a crate. This is the main entry point for the parser. pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> { - let lo = self.span; + let lo = self.token.span; let krate = Ok(ast::Crate { attrs: self.parse_inner_attributes()?, module: self.parse_mod_items(&token::Eof, lo)?, - span: lo.to(self.span), + span: lo.to(self.token.span), }); krate } pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> { - let ret = match self.token { - token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf), - token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf), + let ret = match self.token.kind { + token::Literal(token::Lit { kind: token::Str, symbol, suffix }) => + (symbol, ast::StrStyle::Cooked, suffix), + token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) => + (symbol, ast::StrStyle::Raw(n), suffix), _ => return None }; self.bump(); @@ -8586,24 +7777,43 @@ impl<'a> Parser<'a> { match self.parse_optional_str() { Some((s, style, suf)) => { let sp = self.prev_span; - self.expect_no_suffix(sp, "string literal", suf); + self.expect_no_suffix(sp, "a string literal", suf); Ok((s, style)) } _ => { let msg = "expected string literal"; let mut err = self.fatal(msg); - err.span_label(self.span, msg); + err.span_label(self.token.span, msg); Err(err) } } } + + fn report_invalid_macro_expansion_item(&self) { + self.struct_span_err( + self.prev_span, + "macros that expand to items must be delimited with braces or followed by a semicolon", + ).multipart_suggestion( + "change the delimiters to curly braces", + vec![ + (self.prev_span.with_hi(self.prev_span.lo() + BytePos(1)), String::from(" {")), + (self.prev_span.with_lo(self.prev_span.hi() - BytePos(1)), '}'.to_string()), + ], + Applicability::MaybeIncorrect, + ).span_suggestion( + self.sess.source_map.next_point(self.prev_span), + "add a semicolon", + ';'.to_string(), + Applicability::MaybeIncorrect, + ).emit(); + } } pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, handler: &errors::Handler) { for unmatched in unclosed_delims.iter() { let mut err = handler.struct_span_err(unmatched.found_span, &format!( "incorrect close delimiter: `{}`", - pprust::token_to_string(&token::Token::CloseDelim(unmatched.found_delim)), + pprust::token_kind_to_string(&token::CloseDelim(unmatched.found_delim)), )); err.span_label(unmatched.found_span, "incorrect close delimiter"); if let Some(sp) = unmatched.candidate_span { diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs new file mode 100644 index 00000000000..e619fd17fb5 --- /dev/null +++ b/src/libsyntax/parse/tests.rs @@ -0,0 +1,339 @@ +use super::*; + +use crate::ast::{self, Name, PatKind}; +use crate::attr::first_attr_value_str_by_name; +use crate::parse::{ParseSess, PResult}; +use crate::parse::new_parser_from_source_str; +use crate::parse::token::Token; +use crate::print::pprust::item_to_string; +use crate::ptr::P; +use crate::source_map::FilePathMapping; +use crate::symbol::{kw, sym}; +use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; +use crate::tokenstream::{DelimSpan, TokenTree, TokenStream}; +use crate::with_default_globals; +use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; + +use std::path::PathBuf; + +/// Parses an item. +/// +/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` +/// when a syntax error occurred. +fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) + -> PResult<'_, Option<P<ast::Item>>> { + new_parser_from_source_str(sess, name, source).parse_item() +} + +// produce a syntax_pos::span +fn sp(a: u32, b: u32) -> Span { + Span::new(BytePos(a), BytePos(b), NO_EXPANSION) +} + +/// Parse a string, return an expr +fn string_to_expr(source_str : String) -> P<ast::Expr> { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_expr() + }) +} + +/// Parse a string, return an item +fn string_to_item(source_str : String) -> Option<P<ast::Item>> { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_item() + }) +} + +#[should_panic] +#[test] fn bad_path_expr_1() { + with_default_globals(|| { + string_to_expr("::abc::def::return".to_string()); + }) +} + +// check the token-tree-ization of macros +#[test] +fn string_to_tts_macro () { + with_default_globals(|| { + let tts: Vec<_> = + string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); + let tts: &[TokenTree] = &tts[..]; + + match tts { + [ + TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), + TokenTree::Token(Token { kind: token::Not, .. }), + TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), + TokenTree::Delimited(_, macro_delim, macro_tts) + ] + if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => { + let tts = ¯o_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Delimited(_, first_delim, first_tts), + TokenTree::Token(Token { kind: token::FatArrow, .. }), + TokenTree::Delimited(_, second_delim, second_tts), + ] + if macro_delim == &token::Paren => { + let tts = &first_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if first_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), + } + let tts = &second_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if second_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), + } + }, + _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), + } + }, + _ => panic!("value: {:?}",tts), + } + }) +} + +#[test] +fn string_to_tts_1() { + with_default_globals(|| { + let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); + + let expected = TokenStream::new(vec![ + TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), + TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(5, 6), sp(13, 14)), + token::DelimToken::Paren, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), + TokenTree::token(token::Colon, sp(8, 9)).into(), + TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), + ]).into(), + ).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(15, 16), sp(20, 21)), + token::DelimToken::Brace, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), + TokenTree::token(token::Semi, sp(18, 19)).into(), + ]).into(), + ).into() + ]); + + assert_eq!(tts, expected); + }) +} + +#[test] fn parse_use() { + with_default_globals(|| { + let use_s = "use foo::bar::baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + + let use_s = "use foo::bar as baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + }) +} + +#[test] fn parse_extern_crate() { + with_default_globals(|| { + let ex_s = "extern crate foo;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + + let ex_s = "extern crate foo as bar;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + }) +} + +fn get_spans_of_pat_idents(src: &str) -> Vec<Span> { + let item = string_to_item(src.to_string()).unwrap(); + + struct PatIdentVisitor { + spans: Vec<Span> + } + impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { + fn visit_pat(&mut self, p: &'a ast::Pat) { + match p.node { + PatKind::Ident(_ , ref spannedident, _) => { + self.spans.push(spannedident.span.clone()); + } + _ => { + crate::visit::walk_pat(self, p); + } + } + } + } + let mut v = PatIdentVisitor { spans: Vec::new() }; + crate::visit::walk_item(&mut v, &item); + return v.spans; +} + +#[test] fn span_of_self_arg_pat_idents_are_correct() { + with_default_globals(|| { + + let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", + "impl z { fn a (&mut self, &myarg: i32) {} }", + "impl z { fn a (&'a self, &myarg: i32) {} }", + "impl z { fn a (self, &myarg: i32) {} }", + "impl z { fn a (self: Foo, &myarg: i32) {} }", + ]; + + for &src in &srcs { + let spans = get_spans_of_pat_idents(src); + let (lo, hi) = (spans[0].lo(), spans[0].hi()); + assert!("self" == &src[lo.to_usize()..hi.to_usize()], + "\"{}\" != \"self\". src=\"{}\"", + &src[lo.to_usize()..hi.to_usize()], src) + } + }) +} + +#[test] fn parse_exprs () { + with_default_globals(|| { + // just make sure that they parse.... + string_to_expr("3 + 4".to_string()); + string_to_expr("a::z.froob(b,&(987+3))".to_string()); + }) +} + +#[test] fn attrs_fix_bug () { + with_default_globals(|| { + string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) + -> Result<Box<Writer>, String> { +#[cfg(windows)] +fn wb() -> c_int { + (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int +} + +#[cfg(unix)] +fn wb() -> c_int { O_WRONLY as c_int } + +let mut fflags: c_int = wb(); +}".to_string()); + }) +} + +#[test] fn crlf_doc_comments() { + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + + let name_1 = FileName::Custom("crlf_source_1".to_string()); + let source = "/// doc comment\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_1, source, &sess) + .unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/// doc comment"); + + let name_2 = FileName::Custom("crlf_source_2".to_string()); + let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_2, source, &sess) + .unwrap().unwrap(); + let docs = item.attrs.iter().filter(|a| a.path == sym::doc) + .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>(); + let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; + assert_eq!(&docs[..], b); + + let name_3 = FileName::Custom("clrf_source_3".to_string()); + let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); + }); +} + +#[test] +fn ttdelim_span() { + fn parse_expr_from_source_str( + name: FileName, source: String, sess: &ParseSess + ) -> PResult<'_, P<ast::Expr>> { + new_parser_from_source_str(sess, name, source).parse_expr() + } + + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), + "foo!( fn main() { body } )".to_string(), &sess).unwrap(); + + let tts: Vec<_> = match expr.node { + ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(), + _ => panic!("not a macro"), + }; + + let span = tts.iter().rev().next().unwrap().span(); + + match sess.source_map().span_to_snippet(span) { + Ok(s) => assert_eq!(&s[..], "{ body }"), + Err(_) => panic!("could not get snippet"), + } + }); +} + +// This tests that when parsing a string (rather than a file) we don't try +// and read in a file for a module declaration and just parse a stub. +// See `recurse_into_file_modules` in the parser. +#[test] +fn out_of_line_mod() { + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + let item = parse_item_from_source_str( + PathBuf::from("foo").into(), + "mod foo { struct S; mod this_does_not_exist; }".to_owned(), + &sess, + ).unwrap().unwrap(); + + if let ast::ItemKind::Mod(ref m) = item.node { + assert!(m.items.len() == 2); + } else { + panic!(); + } + }); +} + +#[test] +fn eqmodws() { + assert_eq!(matches_codepattern("",""),true); + assert_eq!(matches_codepattern("","a"),false); + assert_eq!(matches_codepattern("a",""),false); + assert_eq!(matches_codepattern("a","a"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); + assert_eq!(matches_codepattern("a b","a b"),true); + assert_eq!(matches_codepattern("ab","a b"),false); + assert_eq!(matches_codepattern("a b","ab"),true); + assert_eq!(matches_codepattern(" a b","ab"),true); +} + +#[test] +fn pattern_whitespace() { + assert_eq!(matches_codepattern("","\x0C"), false); + assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); +} + +#[test] +fn non_pattern_whitespace() { + // These have the property 'White_Space' but not 'Pattern_White_Space' + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); + assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); +} diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 2fa4f5263fb..be800b4de66 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -1,25 +1,24 @@ pub use BinOpToken::*; pub use Nonterminal::*; pub use DelimToken::*; -pub use Lit::*; -pub use Token::*; +pub use LitKind::*; +pub use TokenKind::*; use crate::ast::{self}; -use crate::parse::ParseSess; +use crate::parse::{parse_stream_from_source_str, ParseSess}; use crate::print::pprust; use crate::ptr::P; -use crate::symbol::keywords; -use crate::syntax::parse::parse_stream_from_source_str; +use crate::symbol::kw; use crate::tokenstream::{self, DelimSpan, TokenStream, TokenTree}; -use syntax_pos::symbol::{self, Symbol}; -use syntax_pos::{self, Span, FileName}; +use syntax_pos::symbol::Symbol; +use syntax_pos::{self, Span, FileName, DUMMY_SP}; use log::info; use std::fmt; use std::mem; #[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert; +use rustc_data_structures::static_assert_size; use rustc_data_structures::sync::Lrc; #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] @@ -59,84 +58,143 @@ impl DelimToken { } } -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum Lit { - Byte(ast::Name), - Char(ast::Name), - Err(ast::Name), - Integer(ast::Name), - Float(ast::Name), - Str_(ast::Name), - StrRaw(ast::Name, u16), /* raw str delimited by n hash symbols */ - ByteStr(ast::Name), - ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */ +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum LitKind { + Bool, // AST only, must never appear in a `Token` + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols + Err, } -impl Lit { - crate fn literal_name(&self) -> &'static str { - match *self { - Byte(_) => "byte literal", - Char(_) => "char literal", - Err(_) => "invalid literal", - Integer(_) => "integer literal", - Float(_) => "float literal", - Str_(_) | StrRaw(..) => "string literal", - ByteStr(_) | ByteStrRaw(..) => "byte string literal" +/// A literal token. +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Lit { + pub kind: LitKind, + pub symbol: Symbol, + pub suffix: Option<Symbol>, +} + +impl fmt::Display for Lit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Lit { kind, symbol, suffix } = *self; + match kind { + Byte => write!(f, "b'{}'", symbol)?, + Char => write!(f, "'{}'", symbol)?, + Str => write!(f, "\"{}\"", symbol)?, + StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + ByteStr => write!(f, "b\"{}\"", symbol)?, + ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + Integer | + Float | + Bool | + Err => write!(f, "{}", symbol)?, } + + if let Some(suffix) = suffix { + write!(f, "{}", suffix)?; + } + + Ok(()) } +} - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - fn probably_equal_for_proc_macro(&self, other: &Lit) -> bool { - mem::discriminant(self) == mem::discriminant(other) +impl LitKind { + /// An English article for the literal token kind. + crate fn article(self) -> &'static str { + match self { + Integer | Err => "an", + _ => "a", + } + } + + crate fn descr(self) -> &'static str { + match self { + Bool => panic!("literal token contains `Lit::Bool`"), + Byte => "byte", + Char => "char", + Integer => "integer", + Float => "float", + Str | StrRaw(..) => "string", + ByteStr | ByteStrRaw(..) => "byte string", + Err => "error", + } + } + + crate fn may_have_suffix(self) -> bool { + match self { + Integer | Float | Err => true, + _ => false, + } } } -pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: Token = Ident(ident, is_raw); +impl Lit { + pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { + Lit { kind, symbol, suffix } + } +} + +pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || [ - keywords::Async.name(), - keywords::Do.name(), - keywords::Box.name(), - keywords::Break.name(), - keywords::Continue.name(), - keywords::False.name(), - keywords::For.name(), - keywords::If.name(), - keywords::Loop.name(), - keywords::Match.name(), - keywords::Move.name(), - keywords::Return.name(), - keywords::True.name(), - keywords::Unsafe.name(), - keywords::While.name(), - keywords::Yield.name(), - keywords::Static.name(), - ].contains(&ident.name) + kw::Async, + + // FIXME: remove when `await!(..)` syntax is removed + // https://github.com/rust-lang/rust/issues/60610 + kw::Await, + + kw::Do, + kw::Box, + kw::Break, + kw::Continue, + kw::False, + kw::For, + kw::If, + kw::Let, + kw::Loop, + kw::Match, + kw::Move, + kw::Return, + kw::True, + kw::Unsafe, + kw::While, + kw::Yield, + kw::Static, + ].contains(&name) } -fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool { - let ident_token: Token = Ident(ident, is_raw); +fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); !ident_token.is_reserved_ident() || ident_token.is_path_segment_keyword() || [ - keywords::Underscore.name(), - keywords::For.name(), - keywords::Impl.name(), - keywords::Fn.name(), - keywords::Unsafe.name(), - keywords::Extern.name(), - keywords::Typeof.name(), - keywords::Dyn.name(), - ].contains(&ident.name) + kw::Underscore, + kw::For, + kw::Impl, + kw::Fn, + kw::Unsafe, + kw::Extern, + kw::Typeof, + kw::Dyn, + ].contains(&name) } -#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)] -pub enum Token { +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum TokenKind { /* Expression-operator symbols. */ Eq, Lt, @@ -176,11 +234,11 @@ pub enum Token { CloseDelim(DelimToken), /* Literals */ - Literal(Lit, Option<ast::Name>), + Literal(Lit), /* Name components */ - Ident(ast::Ident, /* is_raw */ bool), - Lifetime(ast::Ident), + Ident(ast::Name, /* is_raw */ bool), + Lifetime(ast::Name), Interpolated(Lrc<Nonterminal>), @@ -197,22 +255,69 @@ pub enum Token { /// A comment. Comment, Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), Eof, } -// `Token` is used a lot. Make sure it doesn't unintentionally get bigger. +// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_STATEMENT: mem::size_of::<Token>() == 16); +static_assert_size!(TokenKind, 16); + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +impl TokenKind { + pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { + Literal(Lit::new(kind, symbol, suffix)) + } + + /// Returns tokens that are likely to be typed accidentally instead of the current token. + /// Enables better error recovery when the wrong token is found. + crate fn similar_tokens(&self) -> Option<Vec<TokenKind>> { + match *self { + Comma => Some(vec![Dot, Lt, Semi]), + Semi => Some(vec![Colon, Comma]), + _ => None + } + } +} impl Token { + crate fn new(kind: TokenKind, span: Span) -> Self { + Token { kind, span } + } + + /// Some token that will be thrown away later. + crate fn dummy() -> Self { + Token::new(TokenKind::Whitespace, DUMMY_SP) + } + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> Token { - Ident(ident, ident.is_raw_guess()) + crate fn from_ast_ident(ident: ast::Ident) -> Self { + Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) + } + + /// Return this token by value and leave a dummy token in its place. + crate fn take(&mut self) -> Self { + mem::replace(self, Token::dummy()) + } + + crate fn is_op(&self) -> bool { + match self.kind { + OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | + Ident(..) | Lifetime(..) | Interpolated(..) | + Whitespace | Comment | Shebang(..) | Eof => false, + _ => true, + } } crate fn is_like_plus(&self) -> bool { - match *self { + match self.kind { BinOp(Plus) | BinOpEq(Plus) => true, _ => false, } @@ -220,9 +325,9 @@ impl Token { /// Returns `true` if the token can appear at the start of an expression. crate fn can_begin_expr(&self) -> bool { - match *self { - Ident(ident, is_raw) => - ident_can_begin_expr(ident, is_raw), // value name or keyword + match self.kind { + Ident(name, is_raw) => + ident_can_begin_expr(name, self.span, is_raw), // value name or keyword OpenDelim(..) | // tuple, array or block Literal(..) | // literal Not | // operator not @@ -252,9 +357,9 @@ impl Token { /// Returns `true` if the token can appear at the start of a type. crate fn can_begin_type(&self) -> bool { - match *self { - Ident(ident, is_raw) => - ident_can_begin_type(ident, is_raw), // type name or keyword + match self.kind { + Ident(name, is_raw) => + ident_can_begin_type(name, self.span, is_raw), // type name or keyword OpenDelim(Paren) | // tuple OpenDelim(Bracket) | // array Not | // never @@ -274,8 +379,8 @@ impl Token { } /// Returns `true` if the token can appear at the start of a const param. - pub fn can_begin_const_arg(&self) -> bool { - match self { + crate fn can_begin_const_arg(&self) -> bool { + match self.kind { OpenDelim(Brace) => true, Interpolated(ref nt) => match **nt { NtExpr(..) => true, @@ -289,26 +394,33 @@ impl Token { /// Returns `true` if the token can appear at the start of a generic bound. crate fn can_begin_bound(&self) -> bool { - self.is_path_start() || self.is_lifetime() || self.is_keyword(keywords::For) || + self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || self == &Question || self == &OpenDelim(Paren) } /// Returns `true` if the token is any literal crate fn is_lit(&self) -> bool { - match *self { + match self.kind { Literal(..) => true, _ => false, } } + crate fn expect_lit(&self) -> Lit { + match self.kind { + Literal(lit) => lit, + _=> panic!("`expect_lit` called on non-literal"), + } + } + /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). crate fn can_begin_literal_or_bool(&self) -> bool { - match *self { + match self.kind { Literal(..) => true, BinOp(Minus) => true, - Ident(ident, false) if ident.name == keywords::True.name() => true, - Ident(ident, false) if ident.name == keywords::False.name() => true, + Ident(name, false) if name == kw::True => true, + Ident(name, false) if name == kw::False => true, Interpolated(ref nt) => match **nt { NtLiteral(..) => true, _ => false, @@ -319,8 +431,8 @@ impl Token { /// Returns an identifier if this token is an identifier. pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { - match *self { - Ident(ident, is_raw) => Some((ident, is_raw)), + match self.kind { + Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), Interpolated(ref nt) => match **nt { NtIdent(ident, is_raw) => Some((ident, is_raw)), _ => None, @@ -328,10 +440,11 @@ impl Token { _ => None, } } + /// Returns a lifetime identifier if this token is a lifetime. pub fn lifetime(&self) -> Option<ast::Ident> { - match *self { - Lifetime(ident) => Some(ident), + match self.kind { + Lifetime(name) => Some(ast::Ident::new(name, self.span)), Interpolated(ref nt) => match **nt { NtLifetime(ident) => Some(ident), _ => None, @@ -339,6 +452,7 @@ impl Token { _ => None, } } + /// Returns `true` if the token is an identifier. pub fn is_ident(&self) -> bool { self.ident().is_some() @@ -350,16 +464,13 @@ impl Token { /// Returns `true` if the token is a identifier whose name is the given /// string slice. - crate fn is_ident_named(&self, name: &str) -> bool { - match self.ident() { - Some((ident, _)) => ident.as_str() == name, - None => false - } + crate fn is_ident_named(&self, name: Symbol) -> bool { + self.ident().map_or(false, |(ident, _)| ident.name == name) } /// Returns `true` if the token is an interpolated path. fn is_path(&self) -> bool { - if let Interpolated(ref nt) = *self { + if let Interpolated(ref nt) = self.kind { if let NtPath(..) = **nt { return true; } @@ -367,10 +478,23 @@ impl Token { false } + /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? + /// That is, is this a pre-parsed expression dropped into the token stream + /// (which happens while parsing the result of macro expansion)? + crate fn is_whole_expr(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { + return true; + } + } + + false + } + /// Returns `true` if the token is either the `mut` or `const` keyword. crate fn is_mutability(&self) -> bool { - self.is_keyword(keywords::Mut) || - self.is_keyword(keywords::Const) + self.is_keyword(kw::Mut) || + self.is_keyword(kw::Const) } crate fn is_qpath_start(&self) -> bool { @@ -383,11 +507,11 @@ impl Token { } /// Returns `true` if the token is a given keyword, `kw`. - pub fn is_keyword(&self, kw: keywords::Keyword) -> bool { - self.ident().map(|(ident, is_raw)| ident.name == kw.name() && !is_raw).unwrap_or(false) + pub fn is_keyword(&self, kw: Symbol) -> bool { + self.ident().map(|(id, is_raw)| id.name == kw && !is_raw).unwrap_or(false) } - pub fn is_path_segment_keyword(&self) -> bool { + crate fn is_path_segment_keyword(&self) -> bool { match self.ident() { Some((id, false)) => id.is_path_segment_keyword(), _ => false, @@ -396,7 +520,7 @@ impl Token { // Returns true for reserved identifiers used internally for elided lifetimes, // unnamed method parameters, crate root module, error recovery etc. - pub fn is_special_ident(&self) -> bool { + crate fn is_special_ident(&self) -> bool { match self.ident() { Some((id, false)) => id.is_special(), _ => false, @@ -428,58 +552,52 @@ impl Token { } crate fn glue(self, joint: Token) -> Option<Token> { - Some(match self { - Eq => match joint { + let kind = match self.kind { + Eq => match joint.kind { Eq => EqEq, Gt => FatArrow, _ => return None, }, - Lt => match joint { + Lt => match joint.kind { Eq => Le, Lt => BinOp(Shl), Le => BinOpEq(Shl), BinOp(Minus) => LArrow, _ => return None, }, - Gt => match joint { + Gt => match joint.kind { Eq => Ge, Gt => BinOp(Shr), Ge => BinOpEq(Shr), _ => return None, }, - Not => match joint { + Not => match joint.kind { Eq => Ne, _ => return None, }, - BinOp(op) => match joint { + BinOp(op) => match joint.kind { Eq => BinOpEq(op), BinOp(And) if op == And => AndAnd, BinOp(Or) if op == Or => OrOr, Gt if op == Minus => RArrow, _ => return None, }, - Dot => match joint { + Dot => match joint.kind { Dot => DotDot, DotDot => DotDotDot, _ => return None, }, - DotDot => match joint { + DotDot => match joint.kind { Dot => DotDotDot, Eq => DotDotEq, _ => return None, }, - Colon => match joint { + Colon => match joint.kind { Colon => ModSep, _ => return None, }, - SingleQuote => match joint { - Ident(ident, false) => { - let name = Symbol::intern(&format!("'{}", ident)); - Lifetime(symbol::Ident { - name, - span: ident.span, - }) - } + SingleQuote => match joint.kind { + Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), _ => return None, }, @@ -487,27 +605,19 @@ impl Token { DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Eof => return None, - }) - } + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, + }; - /// Returns tokens that are likely to be typed accidentally instead of the current token. - /// Enables better error recovery when the wrong token is found. - crate fn similar_tokens(&self) -> Option<Vec<Token>> { - match *self { - Comma => Some(vec![Dot, Lt, Semi]), - Semi => Some(vec![Colon, Comma]), - _ => None - } + Some(Token::new(kind, self.span.to(joint.span))) } // See comments in `Nonterminal::to_tokenstream` for why we care about // *probably* equal here rather than actual equality crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { - if mem::discriminant(self) != mem::discriminant(other) { + if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { return false } - match (self, other) { + match (&self.kind, &other.kind) { (&Eq, &Eq) | (&Lt, &Lt) | (&Le, &Le) | @@ -547,14 +657,12 @@ impl Token { (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, - (&Lifetime(a), &Lifetime(b)) => a.name == b.name, - (&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name || - a.name == keywords::DollarCrate.name() || - c.name == keywords::DollarCrate.name()), + (&Literal(a), &Literal(b)) => a == b, - (&Literal(ref a, b), &Literal(ref c, d)) => { - b == d && a.probably_equal_for_proc_macro(c) - } + (&Lifetime(a), &Lifetime(b)) => a == b, + (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || + a == kw::DollarCrate || + c == kw::DollarCrate), (&Interpolated(_), &Interpolated(_)) => false, @@ -563,6 +671,12 @@ impl Token { } } +impl PartialEq<TokenKind> for Token { + fn eq(&self, rhs: &TokenKind) -> bool { + self.kind == *rhs + } +} + #[derive(Clone, RustcEncodable, RustcDecodable)] /// For interpolation during macro expansion. pub enum Nonterminal { @@ -580,14 +694,12 @@ pub enum Nonterminal { NtPath(ast::Path), NtVis(ast::Visibility), NtTT(TokenTree), - // These are not exposed to macros, but are used by quasiquote. - NtArm(ast::Arm), - NtImplItem(ast::ImplItem), + // Used only for passing items to proc macro attributes (they are not + // strictly necessary for that, `Annotatable` can be converted into + // tokens directly, but doing that naively regresses pretty-printing). NtTraitItem(ast::TraitItem), + NtImplItem(ast::ImplItem), NtForeignItem(ast::ForeignItem), - NtGenerics(ast::Generics), - NtWhereClause(ast::WhereClause), - NtArg(ast::Arg), } impl PartialEq for Nonterminal { @@ -620,13 +732,9 @@ impl fmt::Debug for Nonterminal { NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtTT(..) => f.pad("NtTT(..)"), - NtArm(..) => f.pad("NtArm(..)"), NtImplItem(..) => f.pad("NtImplItem(..)"), NtTraitItem(..) => f.pad("NtTraitItem(..)"), NtForeignItem(..) => f.pad("NtForeignItem(..)"), - NtGenerics(..) => f.pad("NtGenerics(..)"), - NtWhereClause(..) => f.pad("NtWhereClause(..)"), - NtArg(..) => f.pad("NtArg(..)"), NtVis(..) => f.pad("NtVis(..)"), NtLifetime(..) => f.pad("NtLifetime(..)"), } @@ -658,12 +766,10 @@ impl Nonterminal { prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span) } Nonterminal::NtIdent(ident, is_raw) => { - let token = Token::Ident(ident, is_raw); - Some(TokenTree::Token(ident.span, token).into()) + Some(TokenTree::token(Ident(ident.name, is_raw), ident.span).into()) } Nonterminal::NtLifetime(ident) => { - let token = Token::Lifetime(ident); - Some(TokenTree::Token(ident.span, token).into()) + Some(TokenTree::token(Lifetime(ident.name), ident.span).into()) } Nonterminal::NtTT(ref tt) => { Some(tt.clone().into()) @@ -710,15 +816,6 @@ impl Nonterminal { } } -crate fn is_op(tok: &Token) -> bool { - match *tok { - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | - Ident(..) | Lifetime(..) | Interpolated(..) | - Whitespace | Comment | Shebang(..) | Eof => false, - _ => true, - } -} - fn prepend_attrs(sess: &ParseSess, attrs: &[ast::Attribute], tokens: Option<&tokenstream::TokenStream>, @@ -734,7 +831,7 @@ fn prepend_attrs(sess: &ParseSess, assert_eq!(attr.style, ast::AttrStyle::Outer, "inner attributes should prevent cached tokens from existing"); - let source = pprust::attr_to_string(attr); + let source = pprust::attribute_to_string(attr); let macro_filename = FileName::macro_expansion_source_code(&source); if attr.is_sugared_doc { let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); @@ -748,8 +845,8 @@ fn prepend_attrs(sess: &ParseSess, // For simple paths, push the identifier directly if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { let ident = attr.path.segments[0].ident; - let token = Ident(ident, ident.as_str().starts_with("r#")); - brackets.push(tokenstream::TokenTree::Token(ident.span, token)); + let token = Ident(ident.name, ident.as_str().starts_with("r#")); + brackets.push(tokenstream::TokenTree::token(token, ident.span)); // ... and for more complicated paths, fall back to a reparse hack that // should eventually be removed. @@ -763,7 +860,7 @@ fn prepend_attrs(sess: &ParseSess, // The span we list here for `#` and for `[ ... ]` are both wrong in // that it encompasses more than each token, but it hopefully is "good // enough" for now at least. - builder.push(tokenstream::TokenTree::Token(attr.span, Pound)); + builder.push(tokenstream::TokenTree::token(Pound, attr.span)); let delim_span = DelimSpan::from_single(attr.span); builder.push(tokenstream::TokenTree::Delimited( delim_span, DelimToken::Bracket, brackets.build().into())); diff --git a/src/libsyntax/parse/unescape_error_reporting.rs b/src/libsyntax/parse/unescape_error_reporting.rs new file mode 100644 index 00000000000..7eee07e61a9 --- /dev/null +++ b/src/libsyntax/parse/unescape_error_reporting.rs @@ -0,0 +1,209 @@ +//! Utilities for rendering escape sequence errors as diagnostics. + +use std::ops::Range; +use std::iter::once; + +use rustc_lexer::unescape::{EscapeError, Mode}; +use syntax_pos::{Span, BytePos}; + +use crate::errors::{Handler, Applicability}; + +pub(crate) fn emit_unescape_error( + handler: &Handler, + // interior part of the literal, without quotes + lit: &str, + // full span of the literal, including quotes + span_with_quotes: Span, + mode: Mode, + // range of the error inside `lit` + range: Range<usize>, + error: EscapeError, +) { + log::debug!("emit_unescape_error: {:?}, {:?}, {:?}, {:?}, {:?}", + lit, span_with_quotes, mode, range, error); + let span = { + let Range { start, end } = range; + let (start, end) = (start as u32, end as u32); + let lo = span_with_quotes.lo() + BytePos(start + 1); + let hi = lo + BytePos(end - start); + span_with_quotes + .with_lo(lo) + .with_hi(hi) + }; + let last_char = || { + let c = lit[range.clone()].chars().rev().next().unwrap(); + let span = span.with_lo(span.hi() - BytePos(c.len_utf8() as u32)); + (c, span) + }; + match error { + EscapeError::LoneSurrogateUnicodeEscape => { + handler.struct_span_err(span, "invalid unicode character escape") + .help("unicode escape must not be a surrogate") + .emit(); + } + EscapeError::OutOfRangeUnicodeEscape => { + handler.struct_span_err(span, "invalid unicode character escape") + .help("unicode escape must be at most 10FFFF") + .emit(); + } + EscapeError::MoreThanOneChar => { + handler + .struct_span_err( + span_with_quotes, + "character literal may only contain one codepoint", + ) + .span_suggestion( + span_with_quotes, + "if you meant to write a `str` literal, use double quotes", + format!("\"{}\"", lit), + Applicability::MachineApplicable, + ).emit() + } + EscapeError::EscapeOnlyChar => { + let (c, _span) = last_char(); + + let mut msg = if mode.is_bytes() { + "byte constant must be escaped: " + } else { + "character constant must be escaped: " + }.to_string(); + push_escaped_char(&mut msg, c); + + handler.span_err(span, msg.as_str()) + } + EscapeError::BareCarriageReturn => { + let msg = if mode.in_double_quotes() { + "bare CR not allowed in string, use \\r instead" + } else { + "character constant must be escaped: \\r" + }; + handler.span_err(span, msg); + } + EscapeError::BareCarriageReturnInRawString => { + assert!(mode.in_double_quotes()); + let msg = "bare CR not allowed in raw string"; + handler.span_err(span, msg); + } + EscapeError::InvalidEscape => { + let (c, span) = last_char(); + + let label = if mode.is_bytes() { + "unknown byte escape" + } else { + "unknown character escape" + }; + let mut msg = label.to_string(); + msg.push_str(": "); + push_escaped_char(&mut msg, c); + + let mut diag = handler.struct_span_err(span, msg.as_str()); + diag.span_label(span, label); + if c == '{' || c == '}' && !mode.is_bytes() { + diag.help("if used in a formatting string, \ + curly braces are escaped with `{{` and `}}`"); + } else if c == '\r' { + diag.help("this is an isolated carriage return; \ + consider checking your editor and version control settings"); + } + diag.emit(); + } + EscapeError::TooShortHexEscape => { + handler.span_err(span, "numeric character escape is too short") + } + EscapeError::InvalidCharInHexEscape | EscapeError::InvalidCharInUnicodeEscape => { + let (c, span) = last_char(); + + let mut msg = if error == EscapeError::InvalidCharInHexEscape { + "invalid character in numeric character escape: " + } else { + "invalid character in unicode escape: " + }.to_string(); + push_escaped_char(&mut msg, c); + + handler.span_err(span, msg.as_str()) + } + EscapeError::NonAsciiCharInByte => { + assert!(mode.is_bytes()); + let (_c, span) = last_char(); + handler.span_err(span, "byte constant must be ASCII. \ + Use a \\xHH escape for a non-ASCII byte") + } + EscapeError::NonAsciiCharInByteString => { + assert!(mode.is_bytes()); + let (_c, span) = last_char(); + handler.span_err(span, "raw byte string must be ASCII") + } + EscapeError::OutOfRangeHexEscape => { + handler.span_err(span, "this form of character escape may only be used \ + with characters in the range [\\x00-\\x7f]") + } + EscapeError::LeadingUnderscoreUnicodeEscape => { + let (_c, span) = last_char(); + handler.span_err(span, "invalid start of unicode escape") + } + EscapeError::OverlongUnicodeEscape => { + handler.span_err(span, "overlong unicode escape (must have at most 6 hex digits)") + } + EscapeError::UnclosedUnicodeEscape => { + handler.span_err(span, "unterminated unicode escape (needed a `}`)") + } + EscapeError::NoBraceInUnicodeEscape => { + let msg = "incorrect unicode escape sequence"; + let mut diag = handler.struct_span_err(span, msg); + + let mut suggestion = "\\u{".to_owned(); + let mut suggestion_len = 0; + let (c, char_span) = last_char(); + let chars = once(c).chain(lit[range.end..].chars()); + for c in chars.take(6).take_while(|c| c.is_digit(16)) { + suggestion.push(c); + suggestion_len += c.len_utf8(); + } + + if suggestion_len > 0 { + suggestion.push('}'); + let lo = char_span.lo(); + let hi = lo + BytePos(suggestion_len as u32); + diag.span_suggestion( + span.with_lo(lo).with_hi(hi), + "format of unicode escape sequences uses braces", + suggestion, + Applicability::MaybeIncorrect, + ); + } else { + diag.span_label(span, msg); + diag.help( + "format of unicode escape sequences is `\\u{...}`", + ); + } + + diag.emit(); + } + EscapeError::UnicodeEscapeInByte => { + handler.span_err(span, "unicode escape sequences cannot be used \ + as a byte or in a byte string") + } + EscapeError::EmptyUnicodeEscape => { + handler.span_err(span, "empty unicode escape (must have at least 1 hex digit)") + } + EscapeError::ZeroChars => { + handler.span_err(span, "empty character literal") + } + EscapeError::LoneSlash => { + handler.span_err(span, "invalid trailing slash in literal") + } + } +} + +/// Pushes a character to a message string for error reporting +pub(crate) fn push_escaped_char(msg: &mut String, c: char) { + match c { + '\u{20}'..='\u{7e}' => { + // Don't escape \, ' or " for user-facing messages + msg.push(c); + } + _ => { + msg.extend(c.escape_default()); + } + } +} diff --git a/src/libsyntax/print/helpers.rs b/src/libsyntax/print/helpers.rs new file mode 100644 index 00000000000..3449e07f456 --- /dev/null +++ b/src/libsyntax/print/helpers.rs @@ -0,0 +1,34 @@ +use std::borrow::Cow; +use crate::print::pp::Printer; + +impl Printer { + pub fn word_space<W: Into<Cow<'static, str>>>(&mut self, w: W) { + self.word(w); + self.space(); + } + + pub fn popen(&mut self) { + self.word("("); + } + + pub fn pclose(&mut self) { + self.word(")"); + } + + pub fn hardbreak_if_not_bol(&mut self) { + if !self.is_beginning_of_line() { + self.hardbreak() + } + } + + pub fn space_if_not_bol(&mut self) { + if !self.is_beginning_of_line() { self.space(); } + } + + pub fn nbsp(&mut self) { self.word(" ") } + + pub fn word_nbsp<S: Into<Cow<'static, str>>>(&mut self, w: S) { + self.word(w); + self.nbsp() + } +} diff --git a/src/libsyntax/print/pp.rs b/src/libsyntax/print/pp.rs index d8a8cbb655b..660e77f77d0 100644 --- a/src/libsyntax/print/pp.rs +++ b/src/libsyntax/print/pp.rs @@ -131,12 +131,11 @@ //! it. //! //! In this implementation (following the paper, again) the SCAN process is the -//! methods called `Printer::pretty_print_*`, and the 'PRINT' process is the +//! methods called `Printer::scan_*`, and the 'PRINT' process is the //! method called `Printer::print`. use std::collections::VecDeque; use std::fmt; -use std::io; use std::borrow::Cow; use log::debug; @@ -164,7 +163,7 @@ pub enum Token { // In practice a string token contains either a `&'static str` or a // `String`. `Cow` is overkill for this because we never modify the data, // but it's more convenient than rolling our own more specialized type. - String(Cow<'static, str>, isize), + String(Cow<'static, str>), Break(BreakToken), Begin(BeginToken), End, @@ -172,7 +171,7 @@ pub enum Token { } impl Token { - pub fn is_eof(&self) -> bool { + crate fn is_eof(&self) -> bool { match *self { Token::Eof => true, _ => false, @@ -195,7 +194,7 @@ impl Token { impl fmt::Display for Token { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { - Token::String(ref s, len) => write!(f, "STR({},{})", s, len), + Token::String(ref s) => write!(f, "STR({},{})", s, s.len()), Token::Break(_) => f.write_str("BREAK"), Token::Begin(_) => f.write_str("BEGIN"), Token::End => f.write_str("END"), @@ -223,25 +222,26 @@ fn buf_str(buf: &[BufEntry], left: usize, right: usize, lim: usize) -> String { } #[derive(Copy, Clone)] -pub enum PrintStackBreak { +enum PrintStackBreak { Fits, Broken(Breaks), } #[derive(Copy, Clone)] -pub struct PrintStackElem { +struct PrintStackElem { offset: isize, pbreak: PrintStackBreak } const SIZE_INFINITY: isize = 0xffff; -pub fn mk_printer<'a>(out: Box<dyn io::Write+'a>, linewidth: usize) -> Printer<'a> { +pub fn mk_printer() -> Printer { + let linewidth = 78; // Yes 55, it makes the ring buffers big enough to never fall behind. let n: usize = 55 * linewidth; debug!("mk_printer {}", linewidth); Printer { - out, + out: String::new(), buf_max_len: n, margin: linewidth as isize, space: linewidth as isize, @@ -258,8 +258,8 @@ pub fn mk_printer<'a>(out: Box<dyn io::Write+'a>, linewidth: usize) -> Printer<' } } -pub struct Printer<'a> { - out: Box<dyn io::Write+'a>, +pub struct Printer { + out: String, buf_max_len: usize, /// Width of lines we're constrained to margin: isize, @@ -300,8 +300,8 @@ impl Default for BufEntry { } } -impl<'a> Printer<'a> { - pub fn last_token(&mut self) -> Token { +impl Printer { + pub fn last_token(&self) -> Token { self.buf[self.right].token.clone() } @@ -310,16 +310,14 @@ impl<'a> Printer<'a> { self.buf[self.right].token = t; } - fn pretty_print_eof(&mut self) -> io::Result<()> { + fn scan_eof(&mut self) { if !self.scan_stack.is_empty() { self.check_stack(0); - self.advance_left()?; + self.advance_left(); } - self.indent(0); - Ok(()) } - fn pretty_print_begin(&mut self, b: BeginToken) -> io::Result<()> { + fn scan_begin(&mut self, b: BeginToken) { if self.scan_stack.is_empty() { self.left_total = 1; self.right_total = 1; @@ -330,27 +328,21 @@ impl<'a> Printer<'a> { } debug!("pp Begin({})/buffer Vec<{},{}>", b.offset, self.left, self.right); - self.buf[self.right] = BufEntry { token: Token::Begin(b), size: -self.right_total }; - let right = self.right; - self.scan_push(right); - Ok(()) + self.scan_push(BufEntry { token: Token::Begin(b), size: -self.right_total }); } - fn pretty_print_end(&mut self) -> io::Result<()> { + fn scan_end(&mut self) { if self.scan_stack.is_empty() { debug!("pp End/print Vec<{},{}>", self.left, self.right); - self.print_end() + self.print_end(); } else { debug!("pp End/buffer Vec<{},{}>", self.left, self.right); self.advance_right(); - self.buf[self.right] = BufEntry { token: Token::End, size: -1 }; - let right = self.right; - self.scan_push(right); - Ok(()) + self.scan_push(BufEntry { token: Token::End, size: -1 }); } } - fn pretty_print_break(&mut self, b: BreakToken) -> io::Result<()> { + fn scan_break(&mut self, b: BreakToken) { if self.scan_stack.is_empty() { self.left_total = 1; self.right_total = 1; @@ -362,29 +354,27 @@ impl<'a> Printer<'a> { debug!("pp Break({})/buffer Vec<{},{}>", b.offset, self.left, self.right); self.check_stack(0); - let right = self.right; - self.scan_push(right); - self.buf[self.right] = BufEntry { token: Token::Break(b), size: -self.right_total }; + self.scan_push(BufEntry { token: Token::Break(b), size: -self.right_total }); self.right_total += b.blank_space; - Ok(()) } - fn pretty_print_string(&mut self, s: Cow<'static, str>, len: isize) -> io::Result<()> { + fn scan_string(&mut self, s: Cow<'static, str>) { if self.scan_stack.is_empty() { debug!("pp String('{}')/print Vec<{},{}>", s, self.left, self.right); - self.print_string(s, len) + self.print_string(s); } else { debug!("pp String('{}')/buffer Vec<{},{}>", s, self.left, self.right); self.advance_right(); - self.buf[self.right] = BufEntry { token: Token::String(s, len), size: len }; + let len = s.len() as isize; + self.buf[self.right] = BufEntry { token: Token::String(s), size: len }; self.right_total += len; - self.check_stream() + self.check_stream(); } } - pub fn check_stream(&mut self) -> io::Result<()> { + fn check_stream(&mut self) { debug!("check_stream Vec<{}, {}> with left_total={}, right_total={}", self.left, self.right, self.left_total, self.right_total); if self.right_total - self.left_total > self.space { @@ -395,32 +385,32 @@ impl<'a> Printer<'a> { let scanned = self.scan_pop_bottom(); self.buf[scanned].size = SIZE_INFINITY; } - self.advance_left()?; + self.advance_left(); if self.left != self.right { - self.check_stream()?; + self.check_stream(); } } - Ok(()) } - pub fn scan_push(&mut self, x: usize) { - debug!("scan_push {}", x); - self.scan_stack.push_front(x); + fn scan_push(&mut self, entry: BufEntry) { + debug!("scan_push {}", self.right); + self.buf[self.right] = entry; + self.scan_stack.push_front(self.right); } - pub fn scan_pop(&mut self) -> usize { + fn scan_pop(&mut self) -> usize { self.scan_stack.pop_front().unwrap() } - pub fn scan_top(&mut self) -> usize { + fn scan_top(&mut self) -> usize { *self.scan_stack.front().unwrap() } - pub fn scan_pop_bottom(&mut self) -> usize { + fn scan_pop_bottom(&mut self) -> usize { self.scan_stack.pop_back().unwrap() } - pub fn advance_right(&mut self) { + fn advance_right(&mut self) { self.right += 1; self.right %= self.buf_max_len; // Extend the buf if necessary. @@ -430,7 +420,7 @@ impl<'a> Printer<'a> { assert_ne!(self.right, self.left); } - pub fn advance_left(&mut self) -> io::Result<()> { + fn advance_left(&mut self) { debug!("advance_left Vec<{},{}>, sizeof({})={}", self.left, self.right, self.left, self.buf[self.left].size); @@ -441,14 +431,15 @@ impl<'a> Printer<'a> { let len = match left { Token::Break(b) => b.blank_space, - Token::String(_, len) => { + Token::String(ref s) => { + let len = s.len() as isize; assert_eq!(len, left_size); len } _ => 0 }; - self.print(left, left_size)?; + self.print(left, left_size); self.left_total += len; @@ -461,30 +452,28 @@ impl<'a> Printer<'a> { left_size = self.buf[self.left].size; } - - Ok(()) } - pub fn check_stack(&mut self, k: isize) { + fn check_stack(&mut self, k: usize) { if !self.scan_stack.is_empty() { let x = self.scan_top(); match self.buf[x].token { Token::Begin(_) => { if k > 0 { - let popped = self.scan_pop(); - self.buf[popped].size = self.buf[x].size + self.right_total; + self.scan_pop(); + self.buf[x].size += self.right_total; self.check_stack(k - 1); } } Token::End => { // paper says + not =, but that makes no sense. - let popped = self.scan_pop(); - self.buf[popped].size = 1; + self.scan_pop(); + self.buf[x].size = 1; self.check_stack(k + 1); } _ => { - let popped = self.scan_pop(); - self.buf[popped].size = self.buf[x].size + self.right_total; + self.scan_pop(); + self.buf[x].size += self.right_total; if k > 0 { self.check_stack(k); } @@ -493,20 +482,19 @@ impl<'a> Printer<'a> { } } - pub fn print_newline(&mut self, amount: isize) -> io::Result<()> { + fn print_newline(&mut self, amount: isize) { debug!("NEWLINE {}", amount); - let ret = write!(self.out, "\n"); + self.out.push('\n'); self.pending_indentation = 0; self.indent(amount); - ret } - pub fn indent(&mut self, amount: isize) { + fn indent(&mut self, amount: isize) { debug!("INDENT {}", amount); self.pending_indentation += amount; } - pub fn get_top(&mut self) -> PrintStackElem { + fn get_top(&mut self) -> PrintStackElem { match self.print_stack.last() { Some(el) => *el, None => PrintStackElem { @@ -516,7 +504,7 @@ impl<'a> Printer<'a> { } } - pub fn print_begin(&mut self, b: BeginToken, l: isize) -> io::Result<()> { + fn print_begin(&mut self, b: BeginToken, l: isize) { if l > self.space { let col = self.margin - self.space + b.offset; debug!("print Begin -> push broken block at col {}", col); @@ -531,63 +519,62 @@ impl<'a> Printer<'a> { pbreak: PrintStackBreak::Fits }); } - Ok(()) } - pub fn print_end(&mut self) -> io::Result<()> { + fn print_end(&mut self) { debug!("print End -> pop End"); - let print_stack = &mut self.print_stack; - assert!(!print_stack.is_empty()); - print_stack.pop().unwrap(); - Ok(()) + self.print_stack.pop().unwrap(); } - pub fn print_break(&mut self, b: BreakToken, l: isize) -> io::Result<()> { + fn print_break(&mut self, b: BreakToken, l: isize) { let top = self.get_top(); match top.pbreak { PrintStackBreak::Fits => { debug!("print Break({}) in fitting block", b.blank_space); self.space -= b.blank_space; self.indent(b.blank_space); - Ok(()) } PrintStackBreak::Broken(Breaks::Consistent) => { debug!("print Break({}+{}) in consistent block", top.offset, b.offset); - let ret = self.print_newline(top.offset + b.offset); + self.print_newline(top.offset + b.offset); self.space = self.margin - (top.offset + b.offset); - ret } PrintStackBreak::Broken(Breaks::Inconsistent) => { if l > self.space { debug!("print Break({}+{}) w/ newline in inconsistent", top.offset, b.offset); - let ret = self.print_newline(top.offset + b.offset); + self.print_newline(top.offset + b.offset); self.space = self.margin - (top.offset + b.offset); - ret } else { debug!("print Break({}) w/o newline in inconsistent", b.blank_space); self.indent(b.blank_space); self.space -= b.blank_space; - Ok(()) } } } } - pub fn print_string(&mut self, s: Cow<'static, str>, len: isize) -> io::Result<()> { + fn print_string(&mut self, s: Cow<'static, str>) { + let len = s.len() as isize; debug!("print String({})", s); // assert!(len <= space); self.space -= len; - while self.pending_indentation > 0 { - write!(self.out, " ")?; - self.pending_indentation -= 1; - } - write!(self.out, "{}", s) + + // Write the pending indent. A more concise way of doing this would be: + // + // write!(self.out, "{: >n$}", "", n = self.pending_indentation as usize)?; + // + // But that is significantly slower. This code is sufficiently hot, and indents can get + // sufficiently large, that the difference is significant on some workloads. + self.out.reserve(self.pending_indentation as usize); + self.out.extend(std::iter::repeat(' ').take(self.pending_indentation as usize)); + self.pending_indentation = 0; + self.out.push_str(&s); } - pub fn print(&mut self, token: Token, l: isize) -> io::Result<()> { + fn print(&mut self, token: Token, l: isize) { debug!("print {} {} (remaining line space={})", token, l, self.space); debug!("{}", buf_str(&self.buf, @@ -598,9 +585,10 @@ impl<'a> Printer<'a> { Token::Begin(b) => self.print_begin(b, l), Token::End => self.print_end(), Token::Break(b) => self.print_break(b, l), - Token::String(s, len) => { + Token::String(s) => { + let len = s.len() as isize; assert_eq!(len, l); - self.print_string(s, len) + self.print_string(s); } Token::Eof => panic!(), // Eof should never get here. } @@ -609,65 +597,65 @@ impl<'a> Printer<'a> { // Convenience functions to talk to the printer. /// "raw box" - pub fn rbox(&mut self, indent: usize, b: Breaks) -> io::Result<()> { - self.pretty_print_begin(BeginToken { + pub fn rbox(&mut self, indent: usize, b: Breaks) { + self.scan_begin(BeginToken { offset: indent as isize, breaks: b }) } /// Inconsistent breaking box - pub fn ibox(&mut self, indent: usize) -> io::Result<()> { + pub fn ibox(&mut self, indent: usize) { self.rbox(indent, Breaks::Inconsistent) } /// Consistent breaking box - pub fn cbox(&mut self, indent: usize) -> io::Result<()> { + pub fn cbox(&mut self, indent: usize) { self.rbox(indent, Breaks::Consistent) } - pub fn break_offset(&mut self, n: usize, off: isize) -> io::Result<()> { - self.pretty_print_break(BreakToken { + pub fn break_offset(&mut self, n: usize, off: isize) { + self.scan_break(BreakToken { offset: off, blank_space: n as isize }) } - pub fn end(&mut self) -> io::Result<()> { - self.pretty_print_end() + pub fn end(&mut self) { + self.scan_end() } - pub fn eof(&mut self) -> io::Result<()> { - self.pretty_print_eof() + pub fn eof(mut self) -> String { + self.scan_eof(); + self.out } - pub fn word<S: Into<Cow<'static, str>>>(&mut self, wrd: S) -> io::Result<()> { + pub fn word<S: Into<Cow<'static, str>>>(&mut self, wrd: S) { let s = wrd.into(); - let len = s.len() as isize; - self.pretty_print_string(s, len) + self.scan_string(s) } - fn spaces(&mut self, n: usize) -> io::Result<()> { + fn spaces(&mut self, n: usize) { self.break_offset(n, 0) } - pub fn zerobreak(&mut self) -> io::Result<()> { + crate fn zerobreak(&mut self) { self.spaces(0) } - pub fn space(&mut self) -> io::Result<()> { + pub fn space(&mut self) { self.spaces(1) } - pub fn hardbreak(&mut self) -> io::Result<()> { + pub fn hardbreak(&mut self) { self.spaces(SIZE_INFINITY as usize) } - pub fn hardbreak_tok_offset(off: isize) -> Token { - Token::Break(BreakToken {offset: off, blank_space: SIZE_INFINITY}) + pub fn is_beginning_of_line(&self) -> bool { + self.last_token().is_eof() || self.last_token().is_hardbreak_tok() } - pub fn hardbreak_tok() -> Token { - Self::hardbreak_tok_offset(0) + pub fn hardbreak_tok_offset(off: isize) -> Token { + Token::Break(BreakToken {offset: off, blank_space: SIZE_INFINITY}) } } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 49e3fad4af0..bda761244d5 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -4,25 +4,28 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; -use crate::parse::token::{self, BinOpToken, Nonterminal, Token}; +use crate::parse::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; use crate::parse::{self, ParseSess}; use crate::print::pp::{self, Breaks}; use crate::print::pp::Breaks::{Consistent, Inconsistent}; use crate::ptr::P; -use crate::std_inject; -use crate::symbol::keywords; +use crate::symbol::{kw, sym}; use crate::tokenstream::{self, TokenStream, TokenTree}; use rustc_target::spec::abi::{self, Abi}; use syntax_pos::{self, BytePos}; -use syntax_pos::{DUMMY_SP, FileName}; +use syntax_pos::{FileName, Span}; -use std::ascii; use std::borrow::Cow; -use std::io::{self, Write, Read}; -use std::iter::Peekable; -use std::vec; + +#[cfg(test)] +mod tests; + +pub enum MacHeader<'a> { + Path(&'a ast::Path), + Keyword(&'static str), +} pub enum AnnNode<'a> { Ident(&'a ast::Ident), @@ -35,8 +38,8 @@ pub enum AnnNode<'a> { } pub trait PpAnn { - fn pre(&self, _state: &mut State<'_>, _node: AnnNode<'_>) -> io::Result<()> { Ok(()) } - fn post(&self, _state: &mut State<'_>, _node: AnnNode<'_>) -> io::Result<()> { Ok(()) } + fn pre(&self, _state: &mut State<'_>, _node: AnnNode<'_>) { } + fn post(&self, _state: &mut State<'_>, _node: AnnNode<'_>) { } } #[derive(Copy, Clone)] @@ -44,123 +47,109 @@ pub struct NoAnn; impl PpAnn for NoAnn {} -pub struct State<'a> { - pub s: pp::Printer<'a>, - cm: Option<&'a SourceMap>, - comments: Option<Vec<comments::Comment> >, - literals: Peekable<vec::IntoIter<comments::Literal>>, - cur_cmnt: usize, - boxes: Vec<pp::Breaks>, - ann: &'a (dyn PpAnn+'a), - is_expanded: bool +pub struct Comments<'a> { + cm: &'a SourceMap, + comments: Vec<comments::Comment>, + current: usize, } -fn rust_printer<'a>(writer: Box<dyn Write+'a>, ann: &'a dyn PpAnn) -> State<'a> { - State { - s: pp::mk_printer(writer, DEFAULT_COLUMNS), - cm: None, - comments: None, - literals: vec![].into_iter().peekable(), - cur_cmnt: 0, - boxes: Vec::new(), - ann, - is_expanded: false +impl<'a> Comments<'a> { + pub fn new( + cm: &'a SourceMap, + sess: &ParseSess, + filename: FileName, + input: String, + ) -> Comments<'a> { + let comments = comments::gather_comments(sess, filename, input); + Comments { + cm, + comments, + current: 0, + } + } + + pub fn next(&self) -> Option<comments::Comment> { + self.comments.get(self.current).cloned() + } + + pub fn trailing_comment( + &mut self, + span: syntax_pos::Span, + next_pos: Option<BytePos>, + ) -> Option<comments::Comment> { + if let Some(cmnt) = self.next() { + if cmnt.style != comments::Trailing { return None; } + let span_line = self.cm.lookup_char_pos(span.hi()); + let comment_line = self.cm.lookup_char_pos(cmnt.pos); + let next = next_pos.unwrap_or_else(|| cmnt.pos + BytePos(1)); + if span.hi() < cmnt.pos && cmnt.pos < next && span_line.line == comment_line.line { + return Some(cmnt); + } + } + + None } } -pub const INDENT_UNIT: usize = 4; +pub struct State<'a> { + pub s: pp::Printer, + comments: Option<Comments<'a>>, + ann: &'a (dyn PpAnn+'a), + is_expanded: bool +} -pub const DEFAULT_COLUMNS: usize = 78; +crate const INDENT_UNIT: usize = 4; /// Requires you to pass an input filename and reader so that -/// it can scan the input text for comments and literals to -/// copy forward. +/// it can scan the input text for comments to copy forward. pub fn print_crate<'a>(cm: &'a SourceMap, sess: &ParseSess, krate: &ast::Crate, filename: FileName, - input: &mut dyn Read, - out: Box<dyn Write+'a>, + input: String, ann: &'a dyn PpAnn, - is_expanded: bool) -> io::Result<()> { - let mut s = State::new_from_input(cm, sess, filename, input, out, ann, is_expanded); + is_expanded: bool) -> String { + let mut s = State { + s: pp::mk_printer(), + comments: Some(Comments::new(cm, sess, filename, input)), + ann, + is_expanded, + }; - if is_expanded && std_inject::injected_crate_name().is_some() { + if is_expanded && sess.injected_crate_name.try_get().is_some() { // We need to print `#![no_std]` (and its feature gate) so that // compiling pretty-printed source won't inject libstd again. // However we don't want these attributes in the AST because // of the feature gate, so we fake them up here. // #![feature(prelude_import)] - let pi_nested = attr::mk_nested_word_item(ast::Ident::from_str("prelude_import")); - let list = attr::mk_list_item(DUMMY_SP, ast::Ident::from_str("feature"), vec![pi_nested]); - let fake_attr = attr::mk_attr_inner(DUMMY_SP, attr::mk_attr_id(), list); - s.print_attribute(&fake_attr)?; + let pi_nested = attr::mk_nested_word_item(ast::Ident::with_empty_ctxt(sym::prelude_import)); + let list = attr::mk_list_item(ast::Ident::with_empty_ctxt(sym::feature), vec![pi_nested]); + let fake_attr = attr::mk_attr_inner(list); + s.print_attribute(&fake_attr); // #![no_std] - let no_std_meta = attr::mk_word_item(ast::Ident::from_str("no_std")); - let fake_attr = attr::mk_attr_inner(DUMMY_SP, attr::mk_attr_id(), no_std_meta); - s.print_attribute(&fake_attr)?; + let no_std_meta = attr::mk_word_item(ast::Ident::with_empty_ctxt(sym::no_std)); + let fake_attr = attr::mk_attr_inner(no_std_meta); + s.print_attribute(&fake_attr); } - s.print_mod(&krate.module, &krate.attrs)?; - s.print_remaining_comments()?; + s.print_mod(&krate.module, &krate.attrs); + s.print_remaining_comments(); s.s.eof() } -impl<'a> State<'a> { - pub fn new_from_input(cm: &'a SourceMap, - sess: &ParseSess, - filename: FileName, - input: &mut dyn Read, - out: Box<dyn Write+'a>, - ann: &'a dyn PpAnn, - is_expanded: bool) -> State<'a> { - let (cmnts, lits) = comments::gather_comments_and_literals(sess, filename, input); - - State::new( - cm, - out, - ann, - Some(cmnts), - // If the code is post expansion, don't use the table of - // literals, since it doesn't correspond with the literals - // in the AST anymore. - if is_expanded { None } else { Some(lits) }, - is_expanded - ) - } - - pub fn new(cm: &'a SourceMap, - out: Box<dyn Write+'a>, - ann: &'a dyn PpAnn, - comments: Option<Vec<comments::Comment>>, - literals: Option<Vec<comments::Literal>>, - is_expanded: bool) -> State<'a> { - State { - s: pp::mk_printer(out, DEFAULT_COLUMNS), - cm: Some(cm), - comments, - literals: literals.unwrap_or_default().into_iter().peekable(), - cur_cmnt: 0, - boxes: Vec::new(), - ann, - is_expanded: is_expanded - } - } -} - pub fn to_string<F>(f: F) -> String where - F: FnOnce(&mut State<'_>) -> io::Result<()>, + F: FnOnce(&mut State<'_>), { - let mut wr = Vec::new(); - { - let ann = NoAnn; - let mut printer = rust_printer(Box::new(&mut wr), &ann); - f(&mut printer).unwrap(); - printer.s.eof().unwrap(); - } - String::from_utf8(wr).unwrap() + let mut printer = State { + s: pp::mk_printer(), + comments: None, + ann: &NoAnn, + is_expanded: false + }; + f(&mut printer); + printer.s.eof() } fn binop_to_string(op: BinOpToken) -> &'static str { @@ -178,7 +167,72 @@ fn binop_to_string(op: BinOpToken) -> &'static str { } } -pub fn token_to_string(tok: &Token) -> String { +pub fn literal_to_string(lit: token::Lit) -> String { + let token::Lit { kind, symbol, suffix } = lit; + let mut out = match kind { + token::Byte => format!("b'{}'", symbol), + token::Char => format!("'{}'", symbol), + token::Str => format!("\"{}\"", symbol), + token::StrRaw(n) => format!("r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol), + token::ByteStr => format!("b\"{}\"", symbol), + token::ByteStrRaw(n) => format!("br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol), + token::Integer | + token::Float | + token::Bool | + token::Err => symbol.to_string(), + }; + + if let Some(suffix) = suffix { + out.push_str(&suffix.as_str()) + } + + out +} + +/// Print an ident from AST, `$crate` is converted into its respective crate name. +pub fn ast_ident_to_string(ident: ast::Ident, is_raw: bool) -> String { + ident_to_string(ident.name, is_raw, Some(ident.span)) +} + +// AST pretty-printer is used as a fallback for turning AST structures into token streams for +// proc macros. Additionally, proc macros may stringify their input and expect it survive the +// stringification (especially true for proc macro derives written between Rust 1.15 and 1.30). +// So we need to somehow pretty-print `$crate` in a way preserving at least some of its +// hygiene data, most importantly name of the crate it refers to. +// As a result we print `$crate` as `crate` if it refers to the local crate +// and as `::other_crate_name` if it refers to some other crate. +// Note, that this is only done if the ident token is printed from inside of AST pretty-pringing, +// but not otherwise. Pretty-printing is the only way for proc macros to discover token contents, +// so we should not perform this lossy conversion if the top level call to the pretty-printer was +// done for a token stream or a single token. +fn ident_to_string(name: ast::Name, is_raw: bool, convert_dollar_crate: Option<Span>) -> String { + if is_raw { + format!("r#{}", name) + } else { + if name == kw::DollarCrate { + if let Some(span) = convert_dollar_crate { + let converted = span.ctxt().dollar_crate_name(); + return if converted.is_path_segment_keyword() { + converted.to_string() + } else { + format!("::{}", converted) + } + } + } + name.to_string() + } +} + +/// Print the token kind precisely, without converting `$crate` into its respective crate name. +pub fn token_kind_to_string(tok: &TokenKind) -> String { + token_kind_to_string_ext(tok, None) +} + +fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>) -> String { match *tok { token::Eq => "=".to_string(), token::Lt => "<".to_string(), @@ -221,33 +275,10 @@ pub fn token_to_string(tok: &Token) -> String { token::SingleQuote => "'".to_string(), /* Literals */ - token::Literal(lit, suf) => { - let mut out = match lit { - token::Byte(b) => format!("b'{}'", b), - token::Char(c) => format!("'{}'", c), - token::Err(c) => format!("'{}'", c), - token::Float(c) | - token::Integer(c) => c.to_string(), - token::Str_(s) => format!("\"{}\"", s), - token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=s), - token::ByteStr(v) => format!("b\"{}\"", v), - token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=s), - }; - - if let Some(s) = suf { - out.push_str(&s.as_str()) - } - - out - } + token::Literal(lit) => literal_to_string(lit), /* Name components */ - token::Ident(s, false) => s.to_string(), - token::Ident(s, true) => format!("r#{}", s), + token::Ident(s, is_raw) => ident_to_string(s, is_raw, convert_dollar_crate), token::Lifetime(s) => s.to_string(), /* Other */ @@ -256,12 +287,23 @@ pub fn token_to_string(tok: &Token) -> String { token::Whitespace => " ".to_string(), token::Comment => "/* */".to_string(), token::Shebang(s) => format!("/* shebang: {}*/", s), + token::Unknown(s) => s.to_string(), token::Interpolated(ref nt) => nonterminal_to_string(nt), } } -pub fn nonterminal_to_string(nt: &Nonterminal) -> String { +/// Print the token precisely, without converting `$crate` into its respective crate name. +pub fn token_to_string(token: &Token) -> String { + token_to_string_ext(token, false) +} + +fn token_to_string_ext(token: &Token, convert_dollar_crate: bool) -> String { + let convert_dollar_crate = if convert_dollar_crate { Some(token.span) } else { None }; + token_kind_to_string_ext(&token.kind, convert_dollar_crate) +} + +crate fn nonterminal_to_string(nt: &Nonterminal) -> String { match *nt { token::NtExpr(ref e) => expr_to_string(e), token::NtMeta(ref e) => meta_item_to_string(e), @@ -271,17 +313,12 @@ pub fn nonterminal_to_string(nt: &Nonterminal) -> String { token::NtBlock(ref e) => block_to_string(e), token::NtStmt(ref e) => stmt_to_string(e), token::NtPat(ref e) => pat_to_string(e), - token::NtIdent(e, false) => ident_to_string(e), - token::NtIdent(e, true) => format!("r#{}", ident_to_string(e)), - token::NtLifetime(e) => ident_to_string(e), + token::NtIdent(e, is_raw) => ast_ident_to_string(e, is_raw), + token::NtLifetime(e) => e.to_string(), token::NtLiteral(ref e) => expr_to_string(e), token::NtTT(ref tree) => tt_to_string(tree.clone()), - token::NtArm(ref e) => arm_to_string(e), token::NtImplItem(ref e) => impl_item_to_string(e), token::NtTraitItem(ref e) => trait_item_to_string(e), - token::NtGenerics(ref e) => generic_params_to_string(&e.params), - token::NtWhereClause(ref e) => where_clause_to_string(e), - token::NtArg(ref e) => arg_to_string(e), token::NtVis(ref e) => vis_to_string(e), token::NtForeignItem(ref e) => foreign_item_to_string(e), } @@ -299,47 +336,35 @@ pub fn pat_to_string(pat: &ast::Pat) -> String { to_string(|s| s.print_pat(pat)) } -pub fn arm_to_string(arm: &ast::Arm) -> String { - to_string(|s| s.print_arm(arm)) -} - pub fn expr_to_string(e: &ast::Expr) -> String { to_string(|s| s.print_expr(e)) } -pub fn lifetime_to_string(lt: &ast::Lifetime) -> String { - to_string(|s| s.print_lifetime(*lt)) -} - pub fn tt_to_string(tt: tokenstream::TokenTree) -> String { - to_string(|s| s.print_tt(tt)) + to_string(|s| s.print_tt(tt, false)) } pub fn tts_to_string(tts: &[tokenstream::TokenTree]) -> String { - to_string(|s| s.print_tts(tts.iter().cloned().collect())) + tokens_to_string(tts.iter().cloned().collect()) } pub fn tokens_to_string(tokens: TokenStream) -> String { - to_string(|s| s.print_tts(tokens)) + to_string(|s| s.print_tts(tokens, false)) } pub fn stmt_to_string(stmt: &ast::Stmt) -> String { to_string(|s| s.print_stmt(stmt)) } -pub fn attr_to_string(attr: &ast::Attribute) -> String { - to_string(|s| s.print_attribute(attr)) -} - pub fn item_to_string(i: &ast::Item) -> String { to_string(|s| s.print_item(i)) } -pub fn impl_item_to_string(i: &ast::ImplItem) -> String { +fn impl_item_to_string(i: &ast::ImplItem) -> String { to_string(|s| s.print_impl_item(i)) } -pub fn trait_item_to_string(i: &ast::TraitItem) -> String { +fn trait_item_to_string(i: &ast::TraitItem) -> String { to_string(|s| s.print_trait_item(i)) } @@ -347,14 +372,6 @@ pub fn generic_params_to_string(generic_params: &[ast::GenericParam]) -> String to_string(|s| s.print_generic_params(generic_params)) } -pub fn where_clause_to_string(i: &ast::WhereClause) -> String { - to_string(|s| s.print_where_clause(i)) -} - -pub fn fn_block_to_string(p: &ast::FnDecl) -> String { - to_string(|s| s.print_fn_block_args(p)) -} - pub fn path_to_string(p: &ast::Path) -> String { to_string(|s| s.print_path(p, false, 0)) } @@ -363,34 +380,16 @@ pub fn path_segment_to_string(p: &ast::PathSegment) -> String { to_string(|s| s.print_path_segment(p, false)) } -pub fn ident_to_string(id: ast::Ident) -> String { - to_string(|s| s.print_ident(id)) -} - pub fn vis_to_string(v: &ast::Visibility) -> String { to_string(|s| s.print_visibility(v)) } -pub fn fun_to_string(decl: &ast::FnDecl, - header: ast::FnHeader, - name: ast::Ident, - generics: &ast::Generics) - -> String { - to_string(|s| { - s.head("")?; - s.print_fn(decl, header, Some(name), - generics, &source_map::dummy_spanned(ast::VisibilityKind::Inherited))?; - s.end()?; // Close the head box - s.end() // Close the outer box - }) -} - -pub fn block_to_string(blk: &ast::Block) -> String { +fn block_to_string(blk: &ast::Block) -> String { to_string(|s| { // containing cbox, will be closed by print-block at } - s.cbox(INDENT_UNIT)?; + s.cbox(INDENT_UNIT); // head-ibox, will be closed by print-block after { - s.ibox(0)?; + s.ibox(0); s.print_block(blk) }) } @@ -407,261 +406,126 @@ pub fn attribute_to_string(attr: &ast::Attribute) -> String { to_string(|s| s.print_attribute(attr)) } -pub fn lit_to_string(l: &ast::Lit) -> String { - to_string(|s| s.print_literal(l)) -} - -pub fn variant_to_string(var: &ast::Variant) -> String { - to_string(|s| s.print_variant(var)) -} - pub fn arg_to_string(arg: &ast::Arg) -> String { to_string(|s| s.print_arg(arg, false)) } -pub fn mac_to_string(arg: &ast::Mac) -> String { - to_string(|s| s.print_mac(arg)) -} - -pub fn foreign_item_to_string(arg: &ast::ForeignItem) -> String { +fn foreign_item_to_string(arg: &ast::ForeignItem) -> String { to_string(|s| s.print_foreign_item(arg)) } -pub fn visibility_qualified(vis: &ast::Visibility, s: &str) -> String { +fn visibility_qualified(vis: &ast::Visibility, s: &str) -> String { format!("{}{}", to_string(|s| s.print_visibility(vis)), s) } -pub trait PrintState<'a> { - fn writer(&mut self) -> &mut pp::Printer<'a>; - fn boxes(&mut self) -> &mut Vec<pp::Breaks>; - fn comments(&mut self) -> &mut Option<Vec<comments::Comment>>; - fn cur_cmnt(&mut self) -> &mut usize; - fn cur_lit(&mut self) -> Option<&comments::Literal>; - fn bump_lit(&mut self) -> Option<comments::Literal>; - - fn word_space<S: Into<Cow<'static, str>>>(&mut self, w: S) -> io::Result<()> { - self.writer().word(w)?; - self.writer().space() - } - - fn popen(&mut self) -> io::Result<()> { self.writer().word("(") } - - fn pclose(&mut self) -> io::Result<()> { self.writer().word(")") } - - fn is_begin(&mut self) -> bool { - match self.writer().last_token() { - pp::Token::Begin(_) => true, - _ => false, - } - } - - fn is_end(&mut self) -> bool { - match self.writer().last_token() { - pp::Token::End => true, - _ => false, - } - } - - // is this the beginning of a line? - fn is_bol(&mut self) -> bool { - self.writer().last_token().is_eof() || self.writer().last_token().is_hardbreak_tok() - } - - fn hardbreak_if_not_bol(&mut self) -> io::Result<()> { - if !self.is_bol() { - self.writer().hardbreak()? - } - Ok(()) - } - - // "raw box" - fn rbox(&mut self, u: usize, b: pp::Breaks) -> io::Result<()> { - self.boxes().push(b); - self.writer().rbox(u, b) +impl std::ops::Deref for State<'_> { + type Target = pp::Printer; + fn deref(&self) -> &Self::Target { + &self.s } +} - fn ibox(&mut self, u: usize) -> io::Result<()> { - self.boxes().push(pp::Breaks::Inconsistent); - self.writer().ibox(u) +impl std::ops::DerefMut for State<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.s } +} - fn end(&mut self) -> io::Result<()> { - self.boxes().pop().unwrap(); - self.writer().end() - } +pub trait PrintState<'a>: std::ops::Deref<Target=pp::Printer> + std::ops::DerefMut { + fn comments(&mut self) -> &mut Option<Comments<'a>>; + fn print_ident(&mut self, ident: ast::Ident); + fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool); - fn commasep<T, F>(&mut self, b: Breaks, elts: &[T], mut op: F) -> io::Result<()> - where F: FnMut(&mut Self, &T) -> io::Result<()>, + fn commasep<T, F>(&mut self, b: Breaks, elts: &[T], mut op: F) + where F: FnMut(&mut Self, &T), { - self.rbox(0, b)?; + self.rbox(0, b); let mut first = true; for elt in elts { - if first { first = false; } else { self.word_space(",")?; } - op(self, elt)?; - } - self.end() - } - - fn next_lit(&mut self, pos: BytePos) -> Option<comments::Literal> { - while let Some(ltrl) = self.cur_lit().cloned() { - if ltrl.pos > pos { break; } - - // we don't need the value here since we're forced to clone cur_lit - // due to lack of NLL. - self.bump_lit(); - if ltrl.pos == pos { - return Some(ltrl); - } + if first { first = false; } else { self.word_space(","); } + op(self, elt); } - - None + self.end(); } - fn maybe_print_comment(&mut self, pos: BytePos) -> io::Result<()> { + fn maybe_print_comment(&mut self, pos: BytePos) { while let Some(ref cmnt) = self.next_comment() { if cmnt.pos < pos { - self.print_comment(cmnt)?; + self.print_comment(cmnt); } else { break } } - Ok(()) } fn print_comment(&mut self, - cmnt: &comments::Comment) -> io::Result<()> { - let r = match cmnt.style { + cmnt: &comments::Comment) { + match cmnt.style { comments::Mixed => { assert_eq!(cmnt.lines.len(), 1); - self.writer().zerobreak()?; - self.writer().word(cmnt.lines[0].clone())?; - self.writer().zerobreak() + self.zerobreak(); + self.word(cmnt.lines[0].clone()); + self.zerobreak() } comments::Isolated => { - self.hardbreak_if_not_bol()?; + self.hardbreak_if_not_bol(); for line in &cmnt.lines { // Don't print empty lines because they will end up as trailing // whitespace if !line.is_empty() { - self.writer().word(line.clone())?; + self.word(line.clone()); } - self.writer().hardbreak()?; + self.hardbreak(); } - Ok(()) } comments::Trailing => { - if !self.is_bol() { - self.writer().word(" ")?; + if !self.is_beginning_of_line() { + self.word(" "); } if cmnt.lines.len() == 1 { - self.writer().word(cmnt.lines[0].clone())?; - self.writer().hardbreak() + self.word(cmnt.lines[0].clone()); + self.hardbreak() } else { - self.ibox(0)?; + self.ibox(0); for line in &cmnt.lines { if !line.is_empty() { - self.writer().word(line.clone())?; + self.word(line.clone()); } - self.writer().hardbreak()?; + self.hardbreak(); } - self.end() + self.end(); } } comments::BlankLine => { // We need to do at least one, possibly two hardbreaks. - let is_semi = match self.writer().last_token() { - pp::Token::String(s, _) => ";" == s, + let twice = match self.last_token() { + pp::Token::String(s) => ";" == s, + pp::Token::Begin(_) => true, + pp::Token::End => true, _ => false }; - if is_semi || self.is_begin() || self.is_end() { - self.writer().hardbreak()?; + if twice { + self.hardbreak(); } - self.writer().hardbreak() + self.hardbreak(); } - }; - match r { - Ok(()) => { - *self.cur_cmnt() = *self.cur_cmnt() + 1; - Ok(()) - } - Err(e) => Err(e), + } + if let Some(cm) = self.comments() { + cm.current += 1; } } fn next_comment(&mut self) -> Option<comments::Comment> { - let cur_cmnt = *self.cur_cmnt(); - match *self.comments() { - Some(ref cmnts) => { - if cur_cmnt < cmnts.len() { - Some(cmnts[cur_cmnt].clone()) - } else { - None - } - } - _ => None - } + self.comments().as_mut().and_then(|c| c.next()) } - fn print_literal(&mut self, lit: &ast::Lit) -> io::Result<()> { - self.maybe_print_comment(lit.span.lo())?; - if let Some(ltrl) = self.next_lit(lit.span.lo()) { - return self.writer().word(ltrl.lit.clone()); - } - match lit.node { - ast::LitKind::Str(st, style) => self.print_string(&st.as_str(), style), - ast::LitKind::Err(st) => { - let st = st.as_str().escape_debug().to_string(); - let mut res = String::with_capacity(st.len() + 2); - res.push('\''); - res.push_str(&st); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Byte(byte) => { - let mut res = String::from("b'"); - res.extend(ascii::escape_default(byte).map(|c| c as char)); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Char(ch) => { - let mut res = String::from("'"); - res.extend(ch.escape_default()); - res.push('\''); - self.writer().word(res) - } - ast::LitKind::Int(i, t) => { - match t { - ast::LitIntType::Signed(st) => { - self.writer().word(st.val_to_string(i as i128)) - } - ast::LitIntType::Unsigned(ut) => { - self.writer().word(ut.val_to_string(i)) - } - ast::LitIntType::Unsuffixed => { - self.writer().word(i.to_string()) - } - } - } - ast::LitKind::Float(ref f, t) => { - self.writer().word(format!("{}{}", &f, t.ty_to_string())) - } - ast::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().get()), - ast::LitKind::Bool(val) => { - if val { self.writer().word("true") } else { self.writer().word("false") } - } - ast::LitKind::ByteStr(ref v) => { - let mut escaped: String = String::new(); - for &ch in v.iter() { - escaped.extend(ascii::escape_default(ch) - .map(|c| c as char)); - } - self.writer().word(format!("b\"{}\"", escaped)) - } - } + fn print_literal(&mut self, lit: &ast::Lit) { + self.maybe_print_comment(lit.span.lo()); + self.word(lit.token.to_string()) } fn print_string(&mut self, st: &str, - style: ast::StrStyle) -> io::Result<()> { + style: ast::StrStyle) { let st = match style { ast::StrStyle::Cooked => { (format!("\"{}\"", st.escape_debug())) @@ -672,32 +536,32 @@ pub trait PrintState<'a> { string=st)) } }; - self.writer().word(st) + self.word(st) } fn print_inner_attributes(&mut self, - attrs: &[ast::Attribute]) -> io::Result<()> { + attrs: &[ast::Attribute]) { self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, true) } fn print_inner_attributes_no_trailing_hardbreak(&mut self, attrs: &[ast::Attribute]) - -> io::Result<()> { + { self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, false) } fn print_outer_attributes(&mut self, - attrs: &[ast::Attribute]) -> io::Result<()> { + attrs: &[ast::Attribute]) { self.print_either_attributes(attrs, ast::AttrStyle::Outer, false, true) } fn print_inner_attributes_inline(&mut self, - attrs: &[ast::Attribute]) -> io::Result<()> { + attrs: &[ast::Attribute]) { self.print_either_attributes(attrs, ast::AttrStyle::Inner, true, true) } fn print_outer_attributes_inline(&mut self, - attrs: &[ast::Attribute]) -> io::Result<()> { + attrs: &[ast::Attribute]) { self.print_either_attributes(attrs, ast::AttrStyle::Outer, true, true) } @@ -705,99 +569,91 @@ pub trait PrintState<'a> { attrs: &[ast::Attribute], kind: ast::AttrStyle, is_inline: bool, - trailing_hardbreak: bool) -> io::Result<()> { + trailing_hardbreak: bool) { let mut count = 0; for attr in attrs { if attr.style == kind { - self.print_attribute_inline(attr, is_inline)?; + self.print_attribute_inline(attr, is_inline); if is_inline { - self.nbsp()?; + self.nbsp(); } count += 1; } } if count > 0 && trailing_hardbreak && !is_inline { - self.hardbreak_if_not_bol()?; - } - Ok(()) - } - - fn print_attribute_path(&mut self, path: &ast::Path) -> io::Result<()> { - for (i, segment) in path.segments.iter().enumerate() { - if i > 0 { - self.writer().word("::")? - } - if segment.ident.name != keywords::PathRoot.name() { - if segment.ident.name == keywords::DollarCrate.name() { - self.print_dollar_crate(segment.ident)?; - } else { - self.writer().word(segment.ident.as_str().get())?; - } - } + self.hardbreak_if_not_bol(); } - Ok(()) } - fn print_attribute(&mut self, attr: &ast::Attribute) -> io::Result<()> { + fn print_attribute(&mut self, attr: &ast::Attribute) { self.print_attribute_inline(attr, false) } fn print_attribute_inline(&mut self, attr: &ast::Attribute, - is_inline: bool) -> io::Result<()> { + is_inline: bool) { if !is_inline { - self.hardbreak_if_not_bol()?; + self.hardbreak_if_not_bol(); } - self.maybe_print_comment(attr.span.lo())?; + self.maybe_print_comment(attr.span.lo()); if attr.is_sugared_doc { - self.writer().word(attr.value_str().unwrap().as_str().get())?; - self.writer().hardbreak() + self.word(attr.value_str().unwrap().as_str().to_string()); + self.hardbreak() } else { match attr.style { - ast::AttrStyle::Inner => self.writer().word("#![")?, - ast::AttrStyle::Outer => self.writer().word("#[")?, - } - if let Some(mi) = attr.meta() { - self.print_meta_item(&mi)? - } else { - self.print_attribute_path(&attr.path)?; - self.writer().space()?; - self.print_tts(attr.tokens.clone())?; + ast::AttrStyle::Inner => self.word("#!["), + ast::AttrStyle::Outer => self.word("#["), + } + self.ibox(0); + match attr.tokens.trees().next() { + Some(TokenTree::Delimited(_, delim, tts)) => { + self.print_mac_common( + Some(MacHeader::Path(&attr.path)), false, None, delim, tts, true, attr.span + ); + } + tree => { + self.print_path(&attr.path, false, 0); + if tree.is_some() { + self.space(); + self.print_tts(attr.tokens.clone(), true); + } + } } - self.writer().word("]") + self.end(); + self.word("]"); } } - fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) -> io::Result<()> { - match item.node { - ast::NestedMetaItemKind::MetaItem(ref mi) => { + fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) { + match item { + ast::NestedMetaItem::MetaItem(ref mi) => { self.print_meta_item(mi) }, - ast::NestedMetaItemKind::Literal(ref lit) => { + ast::NestedMetaItem::Literal(ref lit) => { self.print_literal(lit) } } } - fn print_meta_item(&mut self, item: &ast::MetaItem) -> io::Result<()> { - self.ibox(INDENT_UNIT)?; + fn print_meta_item(&mut self, item: &ast::MetaItem) { + self.ibox(INDENT_UNIT); match item.node { - ast::MetaItemKind::Word => self.print_attribute_path(&item.ident)?, + ast::MetaItemKind::Word => self.print_path(&item.path, false, 0), ast::MetaItemKind::NameValue(ref value) => { - self.print_attribute_path(&item.ident)?; - self.writer().space()?; - self.word_space("=")?; - self.print_literal(value)?; + self.print_path(&item.path, false, 0); + self.space(); + self.word_space("="); + self.print_literal(value); } ast::MetaItemKind::List(ref items) => { - self.print_attribute_path(&item.ident)?; - self.popen()?; + self.print_path(&item.path, false, 0); + self.popen(); self.commasep(Consistent, &items[..], - |s, i| s.print_meta_list_item(i))?; - self.pclose()?; + |s, i| s.print_meta_list_item(i)); + self.pclose(); } } - self.end() + self.end(); } /// This doesn't deserve to be called "pretty" printing, but it should be @@ -807,225 +663,275 @@ pub trait PrintState<'a> { /// appropriate macro, transcribe back into the grammar we just parsed from, /// and then pretty-print the resulting AST nodes (so, e.g., we print /// expression arguments as expressions). It can be done! I think. - fn print_tt(&mut self, tt: tokenstream::TokenTree) -> io::Result<()> { + fn print_tt(&mut self, tt: tokenstream::TokenTree, convert_dollar_crate: bool) { match tt { - TokenTree::Token(_, ref tk) => { - self.writer().word(token_to_string(tk))?; - match *tk { - parse::token::DocComment(..) => { - self.writer().hardbreak() + TokenTree::Token(ref token) => { + self.word(token_to_string_ext(&token, convert_dollar_crate)); + match token.kind { + token::DocComment(..) => { + self.hardbreak() } - _ => Ok(()) + _ => {} } } - TokenTree::Delimited(_, delim, tts) => { - self.writer().word(token_to_string(&token::OpenDelim(delim)))?; - self.writer().space()?; - self.print_tts(tts)?; - self.writer().space()?; - self.writer().word(token_to_string(&token::CloseDelim(delim))) - }, + TokenTree::Delimited(dspan, delim, tts) => { + self.print_mac_common( + None, false, None, delim, tts, convert_dollar_crate, dspan.entire() + ); + } } } - fn print_tts(&mut self, tts: tokenstream::TokenStream) -> io::Result<()> { - self.ibox(0)?; + fn print_tts(&mut self, tts: tokenstream::TokenStream, convert_dollar_crate: bool) { for (i, tt) in tts.into_trees().enumerate() { if i != 0 { - self.writer().space()?; + self.space(); } - self.print_tt(tt)?; + self.print_tt(tt, convert_dollar_crate); } - self.end() } - fn space_if_not_bol(&mut self) -> io::Result<()> { - if !self.is_bol() { self.writer().space()?; } - Ok(()) - } - - fn nbsp(&mut self) -> io::Result<()> { self.writer().word(" ") } - - // AST pretty-printer is used as a fallback for turning AST structures into token streams for - // proc macros. Additionally, proc macros may stringify their input and expect it survive the - // stringification (especially true for proc macro derives written between Rust 1.15 and 1.30). - // So we need to somehow pretty-print `$crate` in paths in a way preserving at least some of - // its hygiene data, most importantly name of the crate it refers to. - // As a result we print `$crate` as `crate` if it refers to the local crate - // and as `::other_crate_name` if it refers to some other crate. - fn print_dollar_crate(&mut self, ident: ast::Ident) -> io::Result<()> { - let name = ident.span.ctxt().dollar_crate_name(); - if !ast::Ident::with_empty_ctxt(name).is_path_segment_keyword() { - self.writer().word("::")?; + fn print_mac_common( + &mut self, + header: Option<MacHeader<'_>>, + has_bang: bool, + ident: Option<ast::Ident>, + delim: DelimToken, + tts: TokenStream, + convert_dollar_crate: bool, + span: Span, + ) { + if delim == DelimToken::Brace { + self.cbox(INDENT_UNIT); + } + match header { + Some(MacHeader::Path(path)) => self.print_path(path, false, 0), + Some(MacHeader::Keyword(kw)) => self.word(kw), + None => {} + } + if has_bang { + self.word("!"); + } + if let Some(ident) = ident { + self.nbsp(); + self.print_ident(ident); + } + match delim { + DelimToken::Brace => { + if header.is_some() || has_bang || ident.is_some() { + self.nbsp(); + } + self.word("{"); + if !tts.is_empty() { + self.space(); + } + } + _ => self.word(token_kind_to_string(&token::OpenDelim(delim))), + } + self.ibox(0); + self.print_tts(tts, convert_dollar_crate); + self.end(); + match delim { + DelimToken::Brace => self.bclose(span), + _ => self.word(token_kind_to_string(&token::CloseDelim(delim))), } - self.writer().word(name.as_str().get()) - } -} - -impl<'a> PrintState<'a> for State<'a> { - fn writer(&mut self) -> &mut pp::Printer<'a> { - &mut self.s - } - - fn boxes(&mut self) -> &mut Vec<pp::Breaks> { - &mut self.boxes - } - - fn comments(&mut self) -> &mut Option<Vec<comments::Comment>> { - &mut self.comments - } - - fn cur_cmnt(&mut self) -> &mut usize { - &mut self.cur_cmnt - } - - fn cur_lit(&mut self) -> Option<&comments::Literal> { - self.literals.peek() } - fn bump_lit(&mut self) -> Option<comments::Literal> { - self.literals.next() - } -} + fn print_path(&mut self, path: &ast::Path, colons_before_params: bool, depth: usize) { + self.maybe_print_comment(path.span.lo()); -impl<'a> State<'a> { - pub fn cbox(&mut self, u: usize) -> io::Result<()> { - self.boxes.push(pp::Breaks::Consistent); - self.s.cbox(u) + for (i, segment) in path.segments[..path.segments.len() - depth].iter().enumerate() { + if i > 0 { + self.word("::") + } + self.print_path_segment(segment, colons_before_params); + } } - pub fn word_nbsp<S: Into<Cow<'static, str>>>(&mut self, w: S) -> io::Result<()> { - self.s.word(w)?; - self.nbsp() + fn print_path_segment(&mut self, segment: &ast::PathSegment, colons_before_params: bool) { + if segment.ident.name != kw::PathRoot { + self.print_ident(segment.ident); + if let Some(ref args) = segment.args { + self.print_generic_args(args, colons_before_params); + } + } } - pub fn head<S: Into<Cow<'static, str>>>(&mut self, w: S) -> io::Result<()> { + fn head<S: Into<Cow<'static, str>>>(&mut self, w: S) { let w = w.into(); // outer-box is consistent - self.cbox(INDENT_UNIT)?; + self.cbox(INDENT_UNIT); // head-box is inconsistent - self.ibox(w.len() + 1)?; + self.ibox(w.len() + 1); // keyword that starts the head if !w.is_empty() { - self.word_nbsp(w)?; + self.word_nbsp(w); } - Ok(()) } - pub fn bopen(&mut self) -> io::Result<()> { - self.s.word("{")?; - self.end() // close the head-box + fn bopen(&mut self) { + self.word("{"); + self.end(); // close the head-box } - pub fn bclose_(&mut self, span: syntax_pos::Span, - indented: usize) -> io::Result<()> { - self.bclose_maybe_open(span, indented, true) - } - pub fn bclose_maybe_open(&mut self, span: syntax_pos::Span, - indented: usize, close_box: bool) -> io::Result<()> { - self.maybe_print_comment(span.hi())?; - self.break_offset_if_not_bol(1, -(indented as isize))?; - self.s.word("}")?; + fn bclose_maybe_open(&mut self, span: syntax_pos::Span, close_box: bool) { + self.maybe_print_comment(span.hi()); + self.break_offset_if_not_bol(1, -(INDENT_UNIT as isize)); + self.word("}"); if close_box { - self.end()?; // close the outer-box + self.end(); // close the outer-box } - Ok(()) - } - pub fn bclose(&mut self, span: syntax_pos::Span) -> io::Result<()> { - self.bclose_(span, INDENT_UNIT) } - pub fn in_cbox(&self) -> bool { - match self.boxes.last() { - Some(&last_box) => last_box == pp::Breaks::Consistent, - None => false - } + fn bclose(&mut self, span: syntax_pos::Span) { + self.bclose_maybe_open(span, true) } - pub fn break_offset_if_not_bol(&mut self, n: usize, - off: isize) -> io::Result<()> { - if !self.is_bol() { - self.s.break_offset(n, off) + fn break_offset_if_not_bol(&mut self, n: usize, off: isize) { + if !self.is_beginning_of_line() { + self.break_offset(n, off) } else { - if off != 0 && self.s.last_token().is_hardbreak_tok() { + if off != 0 && self.last_token().is_hardbreak_tok() { // We do something pretty sketchy here: tuck the nonzero // offset-adjustment we were going to deposit along with the // break into the previous hardbreak. - self.s.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); + self.replace_last_token(pp::Printer::hardbreak_tok_offset(off)); } - Ok(()) } } +} +impl<'a> PrintState<'a> for State<'a> { + fn comments(&mut self) -> &mut Option<Comments<'a>> { + &mut self.comments + } + + fn print_ident(&mut self, ident: ast::Ident) { + self.s.word(ast_ident_to_string(ident, ident.is_raw_guess())); + self.ann.post(self, AnnNode::Ident(&ident)) + } + + fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool) { + if colons_before_params { + self.s.word("::") + } + + match *args { + ast::GenericArgs::AngleBracketed(ref data) => { + self.s.word("<"); + + self.commasep(Inconsistent, &data.args, |s, generic_arg| { + s.print_generic_arg(generic_arg) + }); + + let mut comma = data.args.len() != 0; + + for constraint in data.constraints.iter() { + if comma { + self.word_space(",") + } + self.print_ident(constraint.ident); + self.s.space(); + match constraint.kind { + ast::AssocTyConstraintKind::Equality { ref ty } => { + self.word_space("="); + self.print_type(ty); + } + ast::AssocTyConstraintKind::Bound { ref bounds } => { + self.print_type_bounds(":", &*bounds); + } + } + comma = true; + } + + self.s.word(">") + } + + ast::GenericArgs::Parenthesized(ref data) => { + self.s.word("("); + self.commasep( + Inconsistent, + &data.inputs, + |s, ty| s.print_type(ty)); + self.s.word(")"); + + if let Some(ref ty) = data.output { + self.space_if_not_bol(); + self.word_space("->"); + self.print_type(ty); + } + } + } + } +} + +impl<'a> State<'a> { // Synthesizes a comment that was not textually present in the original source // file. - pub fn synth_comment(&mut self, text: String) -> io::Result<()> { - self.s.word("/*")?; - self.s.space()?; - self.s.word(text)?; - self.s.space()?; + pub fn synth_comment(&mut self, text: String) { + self.s.word("/*"); + self.s.space(); + self.s.word(text); + self.s.space(); self.s.word("*/") } - pub fn commasep_cmnt<T, F, G>(&mut self, + crate fn commasep_cmnt<T, F, G>(&mut self, b: Breaks, elts: &[T], mut op: F, - mut get_span: G) -> io::Result<()> where - F: FnMut(&mut State<'_>, &T) -> io::Result<()>, + mut get_span: G) where + F: FnMut(&mut State<'_>, &T), G: FnMut(&T) -> syntax_pos::Span, { - self.rbox(0, b)?; + self.rbox(0, b); let len = elts.len(); let mut i = 0; for elt in elts { - self.maybe_print_comment(get_span(elt).hi())?; - op(self, elt)?; + self.maybe_print_comment(get_span(elt).hi()); + op(self, elt); i += 1; if i < len { - self.s.word(",")?; + self.s.word(","); self.maybe_print_trailing_comment(get_span(elt), - Some(get_span(&elts[i]).hi()))?; - self.space_if_not_bol()?; + Some(get_span(&elts[i]).hi())); + self.space_if_not_bol(); } } - self.end() + self.end(); } - pub fn commasep_exprs(&mut self, b: Breaks, - exprs: &[P<ast::Expr>]) -> io::Result<()> { + crate fn commasep_exprs(&mut self, b: Breaks, + exprs: &[P<ast::Expr>]) { self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e), |e| e.span) } - pub fn print_mod(&mut self, _mod: &ast::Mod, - attrs: &[ast::Attribute]) -> io::Result<()> { - self.print_inner_attributes(attrs)?; + crate fn print_mod(&mut self, _mod: &ast::Mod, + attrs: &[ast::Attribute]) { + self.print_inner_attributes(attrs); for item in &_mod.items { - self.print_item(item)?; + self.print_item(item); } - Ok(()) } - pub fn print_foreign_mod(&mut self, nmod: &ast::ForeignMod, - attrs: &[ast::Attribute]) -> io::Result<()> { - self.print_inner_attributes(attrs)?; + crate fn print_foreign_mod(&mut self, nmod: &ast::ForeignMod, + attrs: &[ast::Attribute]) { + self.print_inner_attributes(attrs); for item in &nmod.items { - self.print_foreign_item(item)?; + self.print_foreign_item(item); } - Ok(()) } - pub fn print_opt_lifetime(&mut self, lifetime: &Option<ast::Lifetime>) -> io::Result<()> { + crate fn print_opt_lifetime(&mut self, lifetime: &Option<ast::Lifetime>) { if let Some(lt) = *lifetime { - self.print_lifetime(lt)?; - self.nbsp()?; + self.print_lifetime(lt); + self.nbsp(); } - Ok(()) } - pub fn print_generic_arg(&mut self, generic_arg: &GenericArg) -> io::Result<()> { + crate fn print_generic_arg(&mut self, generic_arg: &GenericArg) { match generic_arg { GenericArg::Lifetime(lt) => self.print_lifetime(*lt), GenericArg::Type(ty) => self.print_type(ty), @@ -1033,136 +939,136 @@ impl<'a> State<'a> { } } - pub fn print_type(&mut self, ty: &ast::Ty) -> io::Result<()> { - self.maybe_print_comment(ty.span.lo())?; - self.ibox(0)?; + crate fn print_type(&mut self, ty: &ast::Ty) { + self.maybe_print_comment(ty.span.lo()); + self.ibox(0); match ty.node { ast::TyKind::Slice(ref ty) => { - self.s.word("[")?; - self.print_type(ty)?; - self.s.word("]")?; + self.s.word("["); + self.print_type(ty); + self.s.word("]"); } ast::TyKind::Ptr(ref mt) => { - self.s.word("*")?; + self.s.word("*"); match mt.mutbl { - ast::Mutability::Mutable => self.word_nbsp("mut")?, - ast::Mutability::Immutable => self.word_nbsp("const")?, + ast::Mutability::Mutable => self.word_nbsp("mut"), + ast::Mutability::Immutable => self.word_nbsp("const"), } - self.print_type(&mt.ty)?; + self.print_type(&mt.ty); } ast::TyKind::Rptr(ref lifetime, ref mt) => { - self.s.word("&")?; - self.print_opt_lifetime(lifetime)?; - self.print_mt(mt)?; + self.s.word("&"); + self.print_opt_lifetime(lifetime); + self.print_mt(mt); } ast::TyKind::Never => { - self.s.word("!")?; + self.s.word("!"); }, ast::TyKind::Tup(ref elts) => { - self.popen()?; + self.popen(); self.commasep(Inconsistent, &elts[..], - |s, ty| s.print_type(ty))?; + |s, ty| s.print_type(ty)); if elts.len() == 1 { - self.s.word(",")?; + self.s.word(","); } - self.pclose()?; + self.pclose(); } ast::TyKind::Paren(ref typ) => { - self.popen()?; - self.print_type(typ)?; - self.pclose()?; + self.popen(); + self.print_type(typ); + self.pclose(); } ast::TyKind::BareFn(ref f) => { self.print_ty_fn(f.abi, f.unsafety, &f.decl, None, - &f.generic_params)?; + &f.generic_params); } ast::TyKind::Path(None, ref path) => { - self.print_path(path, false, 0)?; + self.print_path(path, false, 0); } ast::TyKind::Path(Some(ref qself), ref path) => { - self.print_qpath(path, qself, false)? + self.print_qpath(path, qself, false) } ast::TyKind::TraitObject(ref bounds, syntax) => { let prefix = if syntax == ast::TraitObjectSyntax::Dyn { "dyn" } else { "" }; - self.print_type_bounds(prefix, &bounds[..])?; + self.print_type_bounds(prefix, &bounds[..]); } ast::TyKind::ImplTrait(_, ref bounds) => { - self.print_type_bounds("impl", &bounds[..])?; + self.print_type_bounds("impl", &bounds[..]); } ast::TyKind::Array(ref ty, ref length) => { - self.s.word("[")?; - self.print_type(ty)?; - self.s.word("; ")?; - self.print_expr(&length.value)?; - self.s.word("]")?; + self.s.word("["); + self.print_type(ty); + self.s.word("; "); + self.print_expr(&length.value); + self.s.word("]"); } ast::TyKind::Typeof(ref e) => { - self.s.word("typeof(")?; - self.print_expr(&e.value)?; - self.s.word(")")?; + self.s.word("typeof("); + self.print_expr(&e.value); + self.s.word(")"); } ast::TyKind::Infer => { - self.s.word("_")?; + self.s.word("_"); } ast::TyKind::Err => { - self.popen()?; - self.s.word("/*ERROR*/")?; - self.pclose()?; + self.popen(); + self.s.word("/*ERROR*/"); + self.pclose(); } ast::TyKind::ImplicitSelf => { - self.s.word("Self")?; + self.s.word("Self"); } ast::TyKind::Mac(ref m) => { - self.print_mac(m)?; + self.print_mac(m); } ast::TyKind::CVarArgs => { - self.s.word("...")?; + self.s.word("..."); } } - self.end() + self.end(); } - pub fn print_foreign_item(&mut self, - item: &ast::ForeignItem) -> io::Result<()> { - self.hardbreak_if_not_bol()?; - self.maybe_print_comment(item.span.lo())?; - self.print_outer_attributes(&item.attrs)?; + crate fn print_foreign_item(&mut self, + item: &ast::ForeignItem) { + self.hardbreak_if_not_bol(); + self.maybe_print_comment(item.span.lo()); + self.print_outer_attributes(&item.attrs); match item.node { ast::ForeignItemKind::Fn(ref decl, ref generics) => { - self.head("")?; + self.head(""); self.print_fn(decl, ast::FnHeader::default(), Some(item.ident), - generics, &item.vis)?; - self.end()?; // end head-ibox - self.s.word(";")?; - self.end() // end the outer fn box + generics, &item.vis); + self.end(); // end head-ibox + self.s.word(";"); + self.end(); // end the outer fn box } ast::ForeignItemKind::Static(ref t, m) => { - self.head(visibility_qualified(&item.vis, "static"))?; - if m { - self.word_space("mut")?; + self.head(visibility_qualified(&item.vis, "static")); + if m == ast::Mutability::Mutable { + self.word_space("mut"); } - self.print_ident(item.ident)?; - self.word_space(":")?; - self.print_type(t)?; - self.s.word(";")?; - self.end()?; // end the head-ibox - self.end() // end the outer cbox + self.print_ident(item.ident); + self.word_space(":"); + self.print_type(t); + self.s.word(";"); + self.end(); // end the head-ibox + self.end(); // end the outer cbox } ast::ForeignItemKind::Ty => { - self.head(visibility_qualified(&item.vis, "type"))?; - self.print_ident(item.ident)?; - self.s.word(";")?; - self.end()?; // end the head-ibox - self.end() // end the outer cbox + self.head(visibility_qualified(&item.vis, "type")); + self.print_ident(item.ident); + self.s.word(";"); + self.end(); // end the head-ibox + self.end(); // end the outer cbox } ast::ForeignItemKind::Macro(ref m) => { - self.print_mac(m)?; + self.print_mac(m); match m.node.delim { - MacDelimiter::Brace => Ok(()), + MacDelimiter::Brace => {}, _ => self.s.word(";") } } @@ -1174,17 +1080,16 @@ impl<'a> State<'a> { ty: &ast::Ty, default: Option<&ast::Expr>, vis: &ast::Visibility) - -> io::Result<()> { - self.s.word(visibility_qualified(vis, ""))?; - self.word_space("const")?; - self.print_ident(ident)?; - self.word_space(":")?; - self.print_type(ty)?; + self.s.word(visibility_qualified(vis, "")); + self.word_space("const"); + self.print_ident(ident); + self.word_space(":"); + self.print_type(ty); if let Some(expr) = default { - self.s.space()?; - self.word_space("=")?; - self.print_expr(expr)?; + self.s.space(); + self.word_space("="); + self.print_expr(expr); } self.s.word(";") } @@ -1193,140 +1098,141 @@ impl<'a> State<'a> { ident: ast::Ident, bounds: Option<&ast::GenericBounds>, ty: Option<&ast::Ty>) - -> io::Result<()> { - self.word_space("type")?; - self.print_ident(ident)?; + { + self.word_space("type"); + self.print_ident(ident); if let Some(bounds) = bounds { - self.print_type_bounds(":", bounds)?; + self.print_type_bounds(":", bounds); } if let Some(ty) = ty { - self.s.space()?; - self.word_space("=")?; - self.print_type(ty)?; + self.s.space(); + self.word_space("="); + self.print_type(ty); } self.s.word(";") } /// Pretty-print an item - pub fn print_item(&mut self, item: &ast::Item) -> io::Result<()> { - self.hardbreak_if_not_bol()?; - self.maybe_print_comment(item.span.lo())?; - self.print_outer_attributes(&item.attrs)?; - self.ann.pre(self, AnnNode::Item(item))?; + crate fn print_item(&mut self, item: &ast::Item) { + self.hardbreak_if_not_bol(); + self.maybe_print_comment(item.span.lo()); + self.print_outer_attributes(&item.attrs); + self.ann.pre(self, AnnNode::Item(item)); match item.node { ast::ItemKind::ExternCrate(orig_name) => { - self.head(visibility_qualified(&item.vis, "extern crate"))?; + self.head(visibility_qualified(&item.vis, "extern crate")); if let Some(orig_name) = orig_name { - self.print_name(orig_name)?; - self.s.space()?; - self.s.word("as")?; - self.s.space()?; + self.print_name(orig_name); + self.s.space(); + self.s.word("as"); + self.s.space(); } - self.print_ident(item.ident)?; - self.s.word(";")?; - self.end()?; // end inner head-block - self.end()?; // end outer head-block + self.print_ident(item.ident); + self.s.word(";"); + self.end(); // end inner head-block + self.end(); // end outer head-block } ast::ItemKind::Use(ref tree) => { - self.head(visibility_qualified(&item.vis, "use"))?; - self.print_use_tree(tree)?; - self.s.word(";")?; - self.end()?; // end inner head-block - self.end()?; // end outer head-block + self.head(visibility_qualified(&item.vis, "use")); + self.print_use_tree(tree); + self.s.word(";"); + self.end(); // end inner head-block + self.end(); // end outer head-block } ast::ItemKind::Static(ref ty, m, ref expr) => { - self.head(visibility_qualified(&item.vis, "static"))?; + self.head(visibility_qualified(&item.vis, "static")); if m == ast::Mutability::Mutable { - self.word_space("mut")?; + self.word_space("mut"); } - self.print_ident(item.ident)?; - self.word_space(":")?; - self.print_type(ty)?; - self.s.space()?; - self.end()?; // end the head-ibox + self.print_ident(item.ident); + self.word_space(":"); + self.print_type(ty); + self.s.space(); + self.end(); // end the head-ibox - self.word_space("=")?; - self.print_expr(expr)?; - self.s.word(";")?; - self.end()?; // end the outer cbox + self.word_space("="); + self.print_expr(expr); + self.s.word(";"); + self.end(); // end the outer cbox } ast::ItemKind::Const(ref ty, ref expr) => { - self.head(visibility_qualified(&item.vis, "const"))?; - self.print_ident(item.ident)?; - self.word_space(":")?; - self.print_type(ty)?; - self.s.space()?; - self.end()?; // end the head-ibox - - self.word_space("=")?; - self.print_expr(expr)?; - self.s.word(";")?; - self.end()?; // end the outer cbox - } - ast::ItemKind::Fn(ref decl, header, ref typarams, ref body) => { - self.head("")?; + self.head(visibility_qualified(&item.vis, "const")); + self.print_ident(item.ident); + self.word_space(":"); + self.print_type(ty); + self.s.space(); + self.end(); // end the head-ibox + + self.word_space("="); + self.print_expr(expr); + self.s.word(";"); + self.end(); // end the outer cbox + } + ast::ItemKind::Fn(ref decl, header, ref param_names, ref body) => { + self.head(""); self.print_fn( decl, header, Some(item.ident), - typarams, + param_names, &item.vis - )?; - self.s.word(" ")?; - self.print_block_with_attrs(body, &item.attrs)?; + ); + self.s.word(" "); + self.print_block_with_attrs(body, &item.attrs); } ast::ItemKind::Mod(ref _mod) => { - self.head(visibility_qualified(&item.vis, "mod"))?; - self.print_ident(item.ident)?; + self.head(visibility_qualified(&item.vis, "mod")); + self.print_ident(item.ident); if _mod.inline || self.is_expanded { - self.nbsp()?; - self.bopen()?; - self.print_mod(_mod, &item.attrs)?; - self.bclose(item.span)?; + self.nbsp(); + self.bopen(); + self.print_mod(_mod, &item.attrs); + self.bclose(item.span); } else { - self.s.word(";")?; - self.end()?; // end inner head-block - self.end()?; // end outer head-block + self.s.word(";"); + self.end(); // end inner head-block + self.end(); // end outer head-block } } ast::ItemKind::ForeignMod(ref nmod) => { - self.head("extern")?; - self.word_nbsp(nmod.abi.to_string())?; - self.bopen()?; - self.print_foreign_mod(nmod, &item.attrs)?; - self.bclose(item.span)?; + self.head("extern"); + self.word_nbsp(nmod.abi.to_string()); + self.bopen(); + self.print_foreign_mod(nmod, &item.attrs); + self.bclose(item.span); } ast::ItemKind::GlobalAsm(ref ga) => { - self.head(visibility_qualified(&item.vis, "global_asm!"))?; - self.s.word(ga.asm.as_str().get())?; - self.end()?; - } - ast::ItemKind::Ty(ref ty, ref generics) => { - self.head(visibility_qualified(&item.vis, "type"))?; - self.print_ident(item.ident)?; - self.print_generic_params(&generics.params)?; - self.end()?; // end the inner ibox - - self.print_where_clause(&generics.where_clause)?; - self.s.space()?; - self.word_space("=")?; - self.print_type(ty)?; - self.s.word(";")?; - self.end()?; // end the outer ibox - } - ast::ItemKind::Existential(ref bounds, ref generics) => { - self.head(visibility_qualified(&item.vis, "existential type"))?; - self.print_ident(item.ident)?; - self.print_generic_params(&generics.params)?; - self.end()?; // end the inner ibox - - self.print_where_clause(&generics.where_clause)?; - self.s.space()?; - self.print_type_bounds(":", bounds)?; - self.s.word(";")?; - self.end()?; // end the outer ibox + self.head(visibility_qualified(&item.vis, "global_asm!")); + self.s.word(ga.asm.as_str().to_string()); + self.end(); + } + ast::ItemKind::TyAlias(ref ty, ref generics) => { + self.head(visibility_qualified(&item.vis, "type")); + self.print_ident(item.ident); + self.print_generic_params(&generics.params); + self.end(); // end the inner ibox + + self.print_where_clause(&generics.where_clause); + self.s.space(); + self.word_space("="); + self.print_type(ty); + self.s.word(";"); + self.end(); // end the outer ibox + } + ast::ItemKind::OpaqueTy(ref bounds, ref generics) => { + self.head(visibility_qualified(&item.vis, "type")); + self.print_ident(item.ident); + self.word_space("= impl"); + self.print_generic_params(&generics.params); + self.end(); // end the inner ibox + + self.print_where_clause(&generics.where_clause); + self.s.space(); + self.print_type_bounds(":", bounds); + self.s.word(";"); + self.end(); // end the outer ibox } ast::ItemKind::Enum(ref enum_definition, ref params) => { self.print_enum_def( @@ -1335,15 +1241,15 @@ impl<'a> State<'a> { item.ident, item.span, &item.vis - )?; + ); } ast::ItemKind::Struct(ref struct_def, ref generics) => { - self.head(visibility_qualified(&item.vis, "struct"))?; - self.print_struct(struct_def, generics, item.ident, item.span, true)?; + self.head(visibility_qualified(&item.vis, "struct")); + self.print_struct(struct_def, generics, item.ident, item.span, true); } ast::ItemKind::Union(ref struct_def, ref generics) => { - self.head(visibility_qualified(&item.vis, "union"))?; - self.print_struct(struct_def, generics, item.ident, item.span, true)?; + self.head(visibility_qualified(&item.vis, "union")); + self.print_struct(struct_def, generics, item.ident, item.span, true); } ast::ItemKind::Impl(unsafety, polarity, @@ -1352,171 +1258,161 @@ impl<'a> State<'a> { ref opt_trait, ref ty, ref impl_items) => { - self.head("")?; - self.print_visibility(&item.vis)?; - self.print_defaultness(defaultness)?; - self.print_unsafety(unsafety)?; - self.word_nbsp("impl")?; + self.head(""); + self.print_visibility(&item.vis); + self.print_defaultness(defaultness); + self.print_unsafety(unsafety); + self.word_nbsp("impl"); if !generics.params.is_empty() { - self.print_generic_params(&generics.params)?; - self.s.space()?; + self.print_generic_params(&generics.params); + self.s.space(); } if polarity == ast::ImplPolarity::Negative { - self.s.word("!")?; + self.s.word("!"); } if let Some(ref t) = *opt_trait { - self.print_trait_ref(t)?; - self.s.space()?; - self.word_space("for")?; + self.print_trait_ref(t); + self.s.space(); + self.word_space("for"); } - self.print_type(ty)?; - self.print_where_clause(&generics.where_clause)?; + self.print_type(ty); + self.print_where_clause(&generics.where_clause); - self.s.space()?; - self.bopen()?; - self.print_inner_attributes(&item.attrs)?; + self.s.space(); + self.bopen(); + self.print_inner_attributes(&item.attrs); for impl_item in impl_items { - self.print_impl_item(impl_item)?; + self.print_impl_item(impl_item); } - self.bclose(item.span)?; + self.bclose(item.span); } ast::ItemKind::Trait(is_auto, unsafety, ref generics, ref bounds, ref trait_items) => { - self.head("")?; - self.print_visibility(&item.vis)?; - self.print_unsafety(unsafety)?; - self.print_is_auto(is_auto)?; - self.word_nbsp("trait")?; - self.print_ident(item.ident)?; - self.print_generic_params(&generics.params)?; + self.head(""); + self.print_visibility(&item.vis); + self.print_unsafety(unsafety); + self.print_is_auto(is_auto); + self.word_nbsp("trait"); + self.print_ident(item.ident); + self.print_generic_params(&generics.params); let mut real_bounds = Vec::with_capacity(bounds.len()); for b in bounds.iter() { if let GenericBound::Trait(ref ptr, ast::TraitBoundModifier::Maybe) = *b { - self.s.space()?; - self.word_space("for ?")?; - self.print_trait_ref(&ptr.trait_ref)?; + self.s.space(); + self.word_space("for ?"); + self.print_trait_ref(&ptr.trait_ref); } else { real_bounds.push(b.clone()); } } - self.print_type_bounds(":", &real_bounds[..])?; - self.print_where_clause(&generics.where_clause)?; - self.s.word(" ")?; - self.bopen()?; + self.print_type_bounds(":", &real_bounds[..]); + self.print_where_clause(&generics.where_clause); + self.s.word(" "); + self.bopen(); for trait_item in trait_items { - self.print_trait_item(trait_item)?; + self.print_trait_item(trait_item); } - self.bclose(item.span)?; + self.bclose(item.span); } ast::ItemKind::TraitAlias(ref generics, ref bounds) => { - self.head("")?; - self.print_visibility(&item.vis)?; - self.word_nbsp("trait")?; - self.print_ident(item.ident)?; - self.print_generic_params(&generics.params)?; + self.head(""); + self.print_visibility(&item.vis); + self.word_nbsp("trait"); + self.print_ident(item.ident); + self.print_generic_params(&generics.params); let mut real_bounds = Vec::with_capacity(bounds.len()); // FIXME(durka) this seems to be some quite outdated syntax for b in bounds.iter() { if let GenericBound::Trait(ref ptr, ast::TraitBoundModifier::Maybe) = *b { - self.s.space()?; - self.word_space("for ?")?; - self.print_trait_ref(&ptr.trait_ref)?; + self.s.space(); + self.word_space("for ?"); + self.print_trait_ref(&ptr.trait_ref); } else { real_bounds.push(b.clone()); } } - self.nbsp()?; - self.print_type_bounds("=", &real_bounds[..])?; - self.print_where_clause(&generics.where_clause)?; - self.s.word(";")?; + self.nbsp(); + self.print_type_bounds("=", &real_bounds[..]); + self.print_where_clause(&generics.where_clause); + self.s.word(";"); } ast::ItemKind::Mac(ref mac) => { - if item.ident.name == keywords::Invalid.name() { - self.print_mac(mac)?; - match mac.node.delim { - MacDelimiter::Brace => {} - _ => self.s.word(";")?, - } - } else { - self.print_path(&mac.node.path, false, 0)?; - self.s.word("! ")?; - self.print_ident(item.ident)?; - self.cbox(INDENT_UNIT)?; - self.popen()?; - self.print_tts(mac.node.stream())?; - self.pclose()?; - self.s.word(";")?; - self.end()?; - } - } - ast::ItemKind::MacroDef(ref tts) => { - self.s.word("macro_rules! ")?; - self.print_ident(item.ident)?; - self.cbox(INDENT_UNIT)?; - self.popen()?; - self.print_tts(tts.stream())?; - self.pclose()?; - self.s.word(";")?; - self.end()?; + self.print_mac(mac); + match mac.node.delim { + MacDelimiter::Brace => {} + _ => self.s.word(";"), + } + } + ast::ItemKind::MacroDef(ref macro_def) => { + let (kw, has_bang) = + if macro_def.legacy { ("macro_rules", true) } else { ("macro", false) }; + self.print_mac_common( + Some(MacHeader::Keyword(kw)), + has_bang, + Some(item.ident), + DelimToken::Brace, + macro_def.stream(), + true, + item.span, + ); } } self.ann.post(self, AnnNode::Item(item)) } - fn print_trait_ref(&mut self, t: &ast::TraitRef) -> io::Result<()> { + fn print_trait_ref(&mut self, t: &ast::TraitRef) { self.print_path(&t.path, false, 0) } fn print_formal_generic_params( &mut self, generic_params: &[ast::GenericParam] - ) -> io::Result<()> { + ) { if !generic_params.is_empty() { - self.s.word("for")?; - self.print_generic_params(generic_params)?; - self.nbsp()?; + self.s.word("for"); + self.print_generic_params(generic_params); + self.nbsp(); } - Ok(()) } - fn print_poly_trait_ref(&mut self, t: &ast::PolyTraitRef) -> io::Result<()> { - self.print_formal_generic_params(&t.bound_generic_params)?; + fn print_poly_trait_ref(&mut self, t: &ast::PolyTraitRef) { + self.print_formal_generic_params(&t.bound_generic_params); self.print_trait_ref(&t.trait_ref) } - pub fn print_enum_def(&mut self, enum_definition: &ast::EnumDef, + crate fn print_enum_def(&mut self, enum_definition: &ast::EnumDef, generics: &ast::Generics, ident: ast::Ident, span: syntax_pos::Span, - visibility: &ast::Visibility) -> io::Result<()> { - self.head(visibility_qualified(visibility, "enum"))?; - self.print_ident(ident)?; - self.print_generic_params(&generics.params)?; - self.print_where_clause(&generics.where_clause)?; - self.s.space()?; + visibility: &ast::Visibility) { + self.head(visibility_qualified(visibility, "enum")); + self.print_ident(ident); + self.print_generic_params(&generics.params); + self.print_where_clause(&generics.where_clause); + self.s.space(); self.print_variants(&enum_definition.variants, span) } - pub fn print_variants(&mut self, + crate fn print_variants(&mut self, variants: &[ast::Variant], - span: syntax_pos::Span) -> io::Result<()> { - self.bopen()?; + span: syntax_pos::Span) { + self.bopen(); for v in variants { - self.space_if_not_bol()?; - self.maybe_print_comment(v.span.lo())?; - self.print_outer_attributes(&v.node.attrs)?; - self.ibox(INDENT_UNIT)?; - self.print_variant(v)?; - self.s.word(",")?; - self.end()?; - self.maybe_print_trailing_comment(v.span, None)?; + self.space_if_not_bol(); + self.maybe_print_comment(v.span.lo()); + self.print_outer_attributes(&v.node.attrs); + self.ibox(INDENT_UNIT); + self.print_variant(v); + self.s.word(","); + self.end(); + self.maybe_print_trailing_comment(v.span, None); } self.bclose(span) } - pub fn print_visibility(&mut self, vis: &ast::Visibility) -> io::Result<()> { + crate fn print_visibility(&mut self, vis: &ast::Visibility) { match vis.node { ast::VisibilityKind::Public => self.word_nbsp("pub"), ast::VisibilityKind::Crate(sugar) => match sugar { @@ -1531,86 +1427,88 @@ impl<'a> State<'a> { self.word_nbsp(format!("pub(in {})", path)) } } - ast::VisibilityKind::Inherited => Ok(()) + ast::VisibilityKind::Inherited => {} } } - pub fn print_defaultness(&mut self, defaultness: ast::Defaultness) -> io::Result<()> { + crate fn print_defaultness(&mut self, defaultness: ast::Defaultness) { if let ast::Defaultness::Default = defaultness { - self.word_nbsp("default")?; + self.word_nbsp("default"); } - Ok(()) } - pub fn print_struct(&mut self, + crate fn print_struct(&mut self, struct_def: &ast::VariantData, generics: &ast::Generics, ident: ast::Ident, span: syntax_pos::Span, - print_finalizer: bool) -> io::Result<()> { - self.print_ident(ident)?; - self.print_generic_params(&generics.params)?; - if !struct_def.is_struct() { - if struct_def.is_tuple() { - self.popen()?; - self.commasep( - Inconsistent, struct_def.fields(), - |s, field| { - s.maybe_print_comment(field.span.lo())?; - s.print_outer_attributes(&field.attrs)?; - s.print_visibility(&field.vis)?; - s.print_type(&field.ty) - } - )?; - self.pclose()?; - } - self.print_where_clause(&generics.where_clause)?; - if print_finalizer { - self.s.word(";")?; + print_finalizer: bool) { + self.print_ident(ident); + self.print_generic_params(&generics.params); + match struct_def { + ast::VariantData::Tuple(..) | ast::VariantData::Unit(..) => { + if let ast::VariantData::Tuple(..) = struct_def { + self.popen(); + self.commasep( + Inconsistent, struct_def.fields(), + |s, field| { + s.maybe_print_comment(field.span.lo()); + s.print_outer_attributes(&field.attrs); + s.print_visibility(&field.vis); + s.print_type(&field.ty) + } + ); + self.pclose(); + } + self.print_where_clause(&generics.where_clause); + if print_finalizer { + self.s.word(";"); + } + self.end(); + self.end(); // close the outer-box } - self.end()?; - self.end() // close the outer-box - } else { - self.print_where_clause(&generics.where_clause)?; - self.nbsp()?; - self.bopen()?; - self.hardbreak_if_not_bol()?; + ast::VariantData::Struct(..) => { + self.print_where_clause(&generics.where_clause); + self.nbsp(); + self.bopen(); + self.hardbreak_if_not_bol(); - for field in struct_def.fields() { - self.hardbreak_if_not_bol()?; - self.maybe_print_comment(field.span.lo())?; - self.print_outer_attributes(&field.attrs)?; - self.print_visibility(&field.vis)?; - self.print_ident(field.ident.unwrap())?; - self.word_nbsp(":")?; - self.print_type(&field.ty)?; - self.s.word(",")?; - } + for field in struct_def.fields() { + self.hardbreak_if_not_bol(); + self.maybe_print_comment(field.span.lo()); + self.print_outer_attributes(&field.attrs); + self.print_visibility(&field.vis); + self.print_ident(field.ident.unwrap()); + self.word_nbsp(":"); + self.print_type(&field.ty); + self.s.word(","); + } - self.bclose(span) + self.bclose(span) + } } } - pub fn print_variant(&mut self, v: &ast::Variant) -> io::Result<()> { - self.head("")?; + crate fn print_variant(&mut self, v: &ast::Variant) { + self.head(""); let generics = ast::Generics::default(); - self.print_struct(&v.node.data, &generics, v.node.ident, v.span, false)?; + self.print_struct(&v.node.data, &generics, v.node.ident, v.span, false); match v.node.disr_expr { Some(ref d) => { - self.s.space()?; - self.word_space("=")?; + self.s.space(); + self.word_space("="); self.print_expr(&d.value) } - _ => Ok(()) + _ => {} } } - pub fn print_method_sig(&mut self, + crate fn print_method_sig(&mut self, ident: ast::Ident, generics: &ast::Generics, m: &ast::MethodSig, vis: &ast::Visibility) - -> io::Result<()> { + { self.print_fn(&m.decl, m.header, Some(ident), @@ -1618,12 +1516,12 @@ impl<'a> State<'a> { vis) } - pub fn print_trait_item(&mut self, ti: &ast::TraitItem) - -> io::Result<()> { - self.ann.pre(self, AnnNode::SubItem(ti.id))?; - self.hardbreak_if_not_bol()?; - self.maybe_print_comment(ti.span.lo())?; - self.print_outer_attributes(&ti.attrs)?; + crate fn print_trait_item(&mut self, ti: &ast::TraitItem) + { + self.ann.pre(self, AnnNode::SubItem(ti.id)); + self.hardbreak_if_not_bol(); + self.maybe_print_comment(ti.span.lo()); + self.print_outer_attributes(&ti.attrs); match ti.node { ast::TraitItemKind::Const(ref ty, ref default) => { self.print_associated_const( @@ -1631,283 +1529,255 @@ impl<'a> State<'a> { ty, default.as_ref().map(|expr| &**expr), &source_map::respan(ti.span.shrink_to_lo(), ast::VisibilityKind::Inherited), - )?; + ); } ast::TraitItemKind::Method(ref sig, ref body) => { if body.is_some() { - self.head("")?; + self.head(""); } self.print_method_sig( ti.ident, &ti.generics, sig, &source_map::respan(ti.span.shrink_to_lo(), ast::VisibilityKind::Inherited), - )?; + ); if let Some(ref body) = *body { - self.nbsp()?; - self.print_block_with_attrs(body, &ti.attrs)?; + self.nbsp(); + self.print_block_with_attrs(body, &ti.attrs); } else { - self.s.word(";")?; + self.s.word(";"); } } ast::TraitItemKind::Type(ref bounds, ref default) => { self.print_associated_type(ti.ident, Some(bounds), - default.as_ref().map(|ty| &**ty))?; + default.as_ref().map(|ty| &**ty)); } ast::TraitItemKind::Macro(ref mac) => { - self.print_mac(mac)?; + self.print_mac(mac); match mac.node.delim { MacDelimiter::Brace => {} - _ => self.s.word(";")?, + _ => self.s.word(";"), } } } self.ann.post(self, AnnNode::SubItem(ti.id)) } - pub fn print_impl_item(&mut self, ii: &ast::ImplItem) -> io::Result<()> { - self.ann.pre(self, AnnNode::SubItem(ii.id))?; - self.hardbreak_if_not_bol()?; - self.maybe_print_comment(ii.span.lo())?; - self.print_outer_attributes(&ii.attrs)?; - self.print_defaultness(ii.defaultness)?; + crate fn print_impl_item(&mut self, ii: &ast::ImplItem) { + self.ann.pre(self, AnnNode::SubItem(ii.id)); + self.hardbreak_if_not_bol(); + self.maybe_print_comment(ii.span.lo()); + self.print_outer_attributes(&ii.attrs); + self.print_defaultness(ii.defaultness); match ii.node { ast::ImplItemKind::Const(ref ty, ref expr) => { - self.print_associated_const(ii.ident, ty, Some(expr), &ii.vis)?; + self.print_associated_const(ii.ident, ty, Some(expr), &ii.vis); } ast::ImplItemKind::Method(ref sig, ref body) => { - self.head("")?; - self.print_method_sig(ii.ident, &ii.generics, sig, &ii.vis)?; - self.nbsp()?; - self.print_block_with_attrs(body, &ii.attrs)?; + self.head(""); + self.print_method_sig(ii.ident, &ii.generics, sig, &ii.vis); + self.nbsp(); + self.print_block_with_attrs(body, &ii.attrs); } - ast::ImplItemKind::Type(ref ty) => { - self.print_associated_type(ii.ident, None, Some(ty))?; + ast::ImplItemKind::TyAlias(ref ty) => { + self.print_associated_type(ii.ident, None, Some(ty)); } - ast::ImplItemKind::Existential(ref bounds) => { - self.word_space("existential")?; - self.print_associated_type(ii.ident, Some(bounds), None)?; + ast::ImplItemKind::OpaqueTy(ref bounds) => { + self.word_space("type"); + self.print_ident(ii.ident); + self.word_space("= impl"); + self.print_type_bounds(":", bounds); + self.s.word(";"); } ast::ImplItemKind::Macro(ref mac) => { - self.print_mac(mac)?; + self.print_mac(mac); match mac.node.delim { MacDelimiter::Brace => {} - _ => self.s.word(";")?, + _ => self.s.word(";"), } } } self.ann.post(self, AnnNode::SubItem(ii.id)) } - pub fn print_stmt(&mut self, st: &ast::Stmt) -> io::Result<()> { - self.maybe_print_comment(st.span.lo())?; + crate fn print_stmt(&mut self, st: &ast::Stmt) { + self.maybe_print_comment(st.span.lo()); match st.node { ast::StmtKind::Local(ref loc) => { - self.print_outer_attributes(&loc.attrs)?; - self.space_if_not_bol()?; - self.ibox(INDENT_UNIT)?; - self.word_nbsp("let")?; - - self.ibox(INDENT_UNIT)?; - self.print_local_decl(loc)?; - self.end()?; + self.print_outer_attributes(&loc.attrs); + self.space_if_not_bol(); + self.ibox(INDENT_UNIT); + self.word_nbsp("let"); + + self.ibox(INDENT_UNIT); + self.print_local_decl(loc); + self.end(); if let Some(ref init) = loc.init { - self.nbsp()?; - self.word_space("=")?; - self.print_expr(init)?; + self.nbsp(); + self.word_space("="); + self.print_expr(init); } - self.s.word(";")?; - self.end()?; + self.s.word(";"); + self.end(); } - ast::StmtKind::Item(ref item) => self.print_item(item)?, + ast::StmtKind::Item(ref item) => self.print_item(item), ast::StmtKind::Expr(ref expr) => { - self.space_if_not_bol()?; - self.print_expr_outer_attr_style(expr, false)?; + self.space_if_not_bol(); + self.print_expr_outer_attr_style(expr, false); if parse::classify::expr_requires_semi_to_be_stmt(expr) { - self.s.word(";")?; + self.s.word(";"); } } ast::StmtKind::Semi(ref expr) => { - self.space_if_not_bol()?; - self.print_expr_outer_attr_style(expr, false)?; - self.s.word(";")?; + self.space_if_not_bol(); + self.print_expr_outer_attr_style(expr, false); + self.s.word(";"); } ast::StmtKind::Mac(ref mac) => { let (ref mac, style, ref attrs) = **mac; - self.space_if_not_bol()?; - self.print_outer_attributes(attrs)?; - self.print_mac(mac)?; + self.space_if_not_bol(); + self.print_outer_attributes(attrs); + self.print_mac(mac); if style == ast::MacStmtStyle::Semicolon { - self.s.word(";")?; + self.s.word(";"); } } } self.maybe_print_trailing_comment(st.span, None) } - pub fn print_block(&mut self, blk: &ast::Block) -> io::Result<()> { + crate fn print_block(&mut self, blk: &ast::Block) { self.print_block_with_attrs(blk, &[]) } - pub fn print_block_unclosed(&mut self, blk: &ast::Block) -> io::Result<()> { - self.print_block_unclosed_indent(blk, INDENT_UNIT) - } - - pub fn print_block_unclosed_with_attrs(&mut self, blk: &ast::Block, - attrs: &[ast::Attribute]) - -> io::Result<()> { - self.print_block_maybe_unclosed(blk, INDENT_UNIT, attrs, false) + crate fn print_block_unclosed_indent(&mut self, blk: &ast::Block) { + self.print_block_maybe_unclosed(blk, &[], false) } - pub fn print_block_unclosed_indent(&mut self, blk: &ast::Block, - indented: usize) -> io::Result<()> { - self.print_block_maybe_unclosed(blk, indented, &[], false) - } - - pub fn print_block_with_attrs(&mut self, + crate fn print_block_with_attrs(&mut self, blk: &ast::Block, - attrs: &[ast::Attribute]) -> io::Result<()> { - self.print_block_maybe_unclosed(blk, INDENT_UNIT, attrs, true) + attrs: &[ast::Attribute]) { + self.print_block_maybe_unclosed(blk, attrs, true) } - pub fn print_block_maybe_unclosed(&mut self, + crate fn print_block_maybe_unclosed(&mut self, blk: &ast::Block, - indented: usize, attrs: &[ast::Attribute], - close_box: bool) -> io::Result<()> { + close_box: bool) { match blk.rules { - BlockCheckMode::Unsafe(..) => self.word_space("unsafe")?, + BlockCheckMode::Unsafe(..) => self.word_space("unsafe"), BlockCheckMode::Default => () } - self.maybe_print_comment(blk.span.lo())?; - self.ann.pre(self, AnnNode::Block(blk))?; - self.bopen()?; + self.maybe_print_comment(blk.span.lo()); + self.ann.pre(self, AnnNode::Block(blk)); + self.bopen(); - self.print_inner_attributes(attrs)?; + self.print_inner_attributes(attrs); for (i, st) in blk.stmts.iter().enumerate() { match st.node { ast::StmtKind::Expr(ref expr) if i == blk.stmts.len() - 1 => { - self.maybe_print_comment(st.span.lo())?; - self.space_if_not_bol()?; - self.print_expr_outer_attr_style(expr, false)?; - self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi()))?; + self.maybe_print_comment(st.span.lo()); + self.space_if_not_bol(); + self.print_expr_outer_attr_style(expr, false); + self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi())); } - _ => self.print_stmt(st)?, + _ => self.print_stmt(st), } } - self.bclose_maybe_open(blk.span, indented, close_box)?; + self.bclose_maybe_open(blk.span, close_box); self.ann.post(self, AnnNode::Block(blk)) } - fn print_else(&mut self, els: Option<&ast::Expr>) -> io::Result<()> { + /// Print a `let pats = scrutinee` expression. + crate fn print_let(&mut self, pats: &[P<ast::Pat>], scrutinee: &ast::Expr) { + self.s.word("let "); + + self.print_pats(pats); + self.s.space(); + + self.word_space("="); + self.print_expr_cond_paren( + scrutinee, + Self::cond_needs_par(scrutinee) + || parser::needs_par_as_let_scrutinee(scrutinee.precedence().order()) + ) + } + + fn print_else(&mut self, els: Option<&ast::Expr>) { match els { Some(_else) => { match _else.node { - // "another else-if" + // Another `else if` block. ast::ExprKind::If(ref i, ref then, ref e) => { - self.cbox(INDENT_UNIT - 1)?; - self.ibox(0)?; - self.s.word(" else if ")?; - self.print_expr_as_cond(i)?; - self.s.space()?; - self.print_block(then)?; - self.print_else(e.as_ref().map(|e| &**e)) - } - // "another else-if-let" - ast::ExprKind::IfLet(ref pats, ref expr, ref then, ref e) => { - self.cbox(INDENT_UNIT - 1)?; - self.ibox(0)?; - self.s.word(" else if let ")?; - self.print_pats(pats)?; - self.s.space()?; - self.word_space("=")?; - self.print_expr_as_cond(expr)?; - self.s.space()?; - self.print_block(then)?; + self.cbox(INDENT_UNIT - 1); + self.ibox(0); + self.s.word(" else if "); + self.print_expr_as_cond(i); + self.s.space(); + self.print_block(then); self.print_else(e.as_ref().map(|e| &**e)) } - // "final else" + // Final `else` block. ast::ExprKind::Block(ref b, _) => { - self.cbox(INDENT_UNIT - 1)?; - self.ibox(0)?; - self.s.word(" else ")?; + self.cbox(INDENT_UNIT - 1); + self.ibox(0); + self.s.word(" else "); self.print_block(b) } - // BLEAH, constraints would be great here + // Constraints would be great here! _ => { panic!("print_if saw if with weird alternative"); } } } - _ => Ok(()) + _ => {} } } - pub fn print_if(&mut self, test: &ast::Expr, blk: &ast::Block, - elseopt: Option<&ast::Expr>) -> io::Result<()> { - self.head("if")?; - self.print_expr_as_cond(test)?; - self.s.space()?; - self.print_block(blk)?; - self.print_else(elseopt) - } + crate fn print_if(&mut self, test: &ast::Expr, blk: &ast::Block, + elseopt: Option<&ast::Expr>) { + self.head("if"); + + self.print_expr_as_cond(test); + self.s.space(); - pub fn print_if_let(&mut self, pats: &[P<ast::Pat>], expr: &ast::Expr, blk: &ast::Block, - elseopt: Option<&ast::Expr>) -> io::Result<()> { - self.head("if let")?; - self.print_pats(pats)?; - self.s.space()?; - self.word_space("=")?; - self.print_expr_as_cond(expr)?; - self.s.space()?; - self.print_block(blk)?; + self.print_block(blk); self.print_else(elseopt) } - pub fn print_mac(&mut self, m: &ast::Mac) -> io::Result<()> { - self.print_path(&m.node.path, false, 0)?; - self.s.word("!")?; - match m.node.delim { - MacDelimiter::Parenthesis => self.popen()?, - MacDelimiter::Bracket => self.s.word("[")?, - MacDelimiter::Brace => { - self.head("")?; - self.bopen()?; - } - } - self.print_tts(m.node.stream())?; - match m.node.delim { - MacDelimiter::Parenthesis => self.pclose(), - MacDelimiter::Bracket => self.s.word("]"), - MacDelimiter::Brace => self.bclose(m.span), - } + crate fn print_mac(&mut self, m: &ast::Mac) { + self.print_mac_common( + Some(MacHeader::Path(&m.node.path)), + true, + None, + m.node.delim.to_token(), + m.node.stream(), + true, + m.span, + ); } - - fn print_call_post(&mut self, args: &[P<ast::Expr>]) -> io::Result<()> { - self.popen()?; - self.commasep_exprs(Inconsistent, args)?; + fn print_call_post(&mut self, args: &[P<ast::Expr>]) { + self.popen(); + self.commasep_exprs(Inconsistent, args); self.pclose() } - pub fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8) -> io::Result<()> { - let needs_par = expr.precedence().order() < prec; - if needs_par { - self.popen()?; - } - self.print_expr(expr)?; - if needs_par { - self.pclose()?; - } - Ok(()) + crate fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8) { + self.print_expr_cond_paren(expr, expr.precedence().order() < prec) } /// Print an expr using syntax that's acceptable in a condition position, such as the `cond` in /// `if cond { ... }`. - pub fn print_expr_as_cond(&mut self, expr: &ast::Expr) -> io::Result<()> { - let needs_par = match expr.node { + crate fn print_expr_as_cond(&mut self, expr: &ast::Expr) { + self.print_expr_cond_paren(expr, Self::cond_needs_par(expr)) + } + + /// Does `expr` need parenthesis when printed in a condition position? + fn cond_needs_par(expr: &ast::Expr) -> bool { + match expr.node { // These cases need parens due to the parse error observed in #26461: `if return {}` // parses as the erroneous construct `if (return {})`, not `if (return) {}`. ast::ExprKind::Closure(..) | @@ -1915,115 +1785,116 @@ impl<'a> State<'a> { ast::ExprKind::Break(..) => true, _ => parser::contains_exterior_struct_lit(expr), - }; + } + } + /// Print `expr` or `(expr)` when `needs_par` holds. + fn print_expr_cond_paren(&mut self, expr: &ast::Expr, needs_par: bool) { if needs_par { - self.popen()?; + self.popen(); } - self.print_expr(expr)?; + self.print_expr(expr); if needs_par { - self.pclose()?; + self.pclose(); } - Ok(()) } fn print_expr_vec(&mut self, exprs: &[P<ast::Expr>], - attrs: &[Attribute]) -> io::Result<()> { - self.ibox(INDENT_UNIT)?; - self.s.word("[")?; - self.print_inner_attributes_inline(attrs)?; - self.commasep_exprs(Inconsistent, &exprs[..])?; - self.s.word("]")?; - self.end() + attrs: &[Attribute]) { + self.ibox(INDENT_UNIT); + self.s.word("["); + self.print_inner_attributes_inline(attrs); + self.commasep_exprs(Inconsistent, &exprs[..]); + self.s.word("]"); + self.end(); } fn print_expr_repeat(&mut self, element: &ast::Expr, count: &ast::AnonConst, - attrs: &[Attribute]) -> io::Result<()> { - self.ibox(INDENT_UNIT)?; - self.s.word("[")?; - self.print_inner_attributes_inline(attrs)?; - self.print_expr(element)?; - self.word_space(";")?; - self.print_expr(&count.value)?; - self.s.word("]")?; - self.end() + attrs: &[Attribute]) { + self.ibox(INDENT_UNIT); + self.s.word("["); + self.print_inner_attributes_inline(attrs); + self.print_expr(element); + self.word_space(";"); + self.print_expr(&count.value); + self.s.word("]"); + self.end(); } fn print_expr_struct(&mut self, path: &ast::Path, fields: &[ast::Field], wth: &Option<P<ast::Expr>>, - attrs: &[Attribute]) -> io::Result<()> { - self.print_path(path, true, 0)?; - self.s.word("{")?; - self.print_inner_attributes_inline(attrs)?; + attrs: &[Attribute]) { + self.print_path(path, true, 0); + self.s.word("{"); + self.print_inner_attributes_inline(attrs); self.commasep_cmnt( Consistent, &fields[..], |s, field| { - s.ibox(INDENT_UNIT)?; + s.ibox(INDENT_UNIT); if !field.is_shorthand { - s.print_ident(field.ident)?; - s.word_space(":")?; + s.print_ident(field.ident); + s.word_space(":"); } - s.print_expr(&field.expr)?; - s.end() + s.print_expr(&field.expr); + s.end(); }, - |f| f.span)?; + |f| f.span); match *wth { Some(ref expr) => { - self.ibox(INDENT_UNIT)?; + self.ibox(INDENT_UNIT); if !fields.is_empty() { - self.s.word(",")?; - self.s.space()?; + self.s.word(","); + self.s.space(); } - self.s.word("..")?; - self.print_expr(expr)?; - self.end()?; + self.s.word(".."); + self.print_expr(expr); + self.end(); } _ => if !fields.is_empty() { - self.s.word(",")? + self.s.word(",") } } - self.s.word("}")?; - Ok(()) + self.s.word("}"); } fn print_expr_tup(&mut self, exprs: &[P<ast::Expr>], - attrs: &[Attribute]) -> io::Result<()> { - self.popen()?; - self.print_inner_attributes_inline(attrs)?; - self.commasep_exprs(Inconsistent, &exprs[..])?; + attrs: &[Attribute]) { + self.popen(); + self.print_inner_attributes_inline(attrs); + self.commasep_exprs(Inconsistent, &exprs[..]); if exprs.len() == 1 { - self.s.word(",")?; + self.s.word(","); } self.pclose() } fn print_expr_call(&mut self, func: &ast::Expr, - args: &[P<ast::Expr>]) -> io::Result<()> { + args: &[P<ast::Expr>]) { let prec = match func.node { ast::ExprKind::Field(..) => parser::PREC_FORCE_PAREN, _ => parser::PREC_POSTFIX, }; - self.print_expr_maybe_paren(func, prec)?; + self.print_expr_maybe_paren(func, prec); self.print_call_post(args) } fn print_expr_method_call(&mut self, segment: &ast::PathSegment, - args: &[P<ast::Expr>]) -> io::Result<()> { + args: &[P<ast::Expr>]) { let base_args = &args[1..]; - self.print_expr_maybe_paren(&args[0], parser::PREC_POSTFIX)?; - self.s.word(".")?; - self.print_ident(segment.ident)?; + self.print_expr_maybe_paren(&args[0], parser::PREC_POSTFIX); + self.s.word("."); + self.print_ident(segment.ident); if let Some(ref args) = segment.args { - self.print_generic_args(args, true)?; + self.print_generic_args(args, true); } self.print_call_post(base_args) } @@ -2031,7 +1902,7 @@ impl<'a> State<'a> { fn print_expr_binary(&mut self, op: ast::BinOp, lhs: &ast::Expr, - rhs: &ast::Expr) -> io::Result<()> { + rhs: &ast::Expr) { let assoc_op = AssocOp::from_ast_binop(op.node); let prec = assoc_op.precedence() as i8; let fixity = assoc_op.fixity(); @@ -2048,228 +1919,223 @@ impl<'a> State<'a> { // of `(x as i32) < ...`. We need to convince it _not_ to do that. (&ast::ExprKind::Cast { .. }, ast::BinOpKind::Lt) | (&ast::ExprKind::Cast { .. }, ast::BinOpKind::Shl) => parser::PREC_FORCE_PAREN, + // We are given `(let _ = a) OP b`. + // + // - When `OP <= LAnd` we should print `let _ = a OP b` to avoid redundant parens + // as the parser will interpret this as `(let _ = a) OP b`. + // + // - Otherwise, e.g. when we have `(let a = b) < c` in AST, + // parens are required since the parser would interpret `let a = b < c` as + // `let a = (b < c)`. To achieve this, we force parens. + (&ast::ExprKind::Let { .. }, _) if !parser::needs_par_as_let_scrutinee(prec) => { + parser::PREC_FORCE_PAREN + } _ => left_prec, }; - self.print_expr_maybe_paren(lhs, left_prec)?; - self.s.space()?; - self.word_space(op.node.to_string())?; + self.print_expr_maybe_paren(lhs, left_prec); + self.s.space(); + self.word_space(op.node.to_string()); self.print_expr_maybe_paren(rhs, right_prec) } fn print_expr_unary(&mut self, op: ast::UnOp, - expr: &ast::Expr) -> io::Result<()> { - self.s.word(ast::UnOp::to_string(op))?; + expr: &ast::Expr) { + self.s.word(ast::UnOp::to_string(op)); self.print_expr_maybe_paren(expr, parser::PREC_PREFIX) } fn print_expr_addr_of(&mut self, mutability: ast::Mutability, - expr: &ast::Expr) -> io::Result<()> { - self.s.word("&")?; - self.print_mutability(mutability)?; + expr: &ast::Expr) { + self.s.word("&"); + self.print_mutability(mutability); self.print_expr_maybe_paren(expr, parser::PREC_PREFIX) } - pub fn print_expr(&mut self, expr: &ast::Expr) -> io::Result<()> { + crate fn print_expr(&mut self, expr: &ast::Expr) { self.print_expr_outer_attr_style(expr, true) } fn print_expr_outer_attr_style(&mut self, expr: &ast::Expr, - is_inline: bool) -> io::Result<()> { - self.maybe_print_comment(expr.span.lo())?; + is_inline: bool) { + self.maybe_print_comment(expr.span.lo()); let attrs = &expr.attrs; if is_inline { - self.print_outer_attributes_inline(attrs)?; + self.print_outer_attributes_inline(attrs); } else { - self.print_outer_attributes(attrs)?; + self.print_outer_attributes(attrs); } - self.ibox(INDENT_UNIT)?; - self.ann.pre(self, AnnNode::Expr(expr))?; + self.ibox(INDENT_UNIT); + self.ann.pre(self, AnnNode::Expr(expr)); match expr.node { ast::ExprKind::Box(ref expr) => { - self.word_space("box")?; - self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)?; - } - ast::ExprKind::ObsoleteInPlace(ref place, ref expr) => { - let prec = AssocOp::ObsoleteInPlace.precedence() as i8; - self.print_expr_maybe_paren(place, prec + 1)?; - self.s.space()?; - self.word_space("<-")?; - self.print_expr_maybe_paren(expr, prec)?; + self.word_space("box"); + self.print_expr_maybe_paren(expr, parser::PREC_PREFIX); } ast::ExprKind::Array(ref exprs) => { - self.print_expr_vec(&exprs[..], attrs)?; + self.print_expr_vec(&exprs[..], attrs); } ast::ExprKind::Repeat(ref element, ref count) => { - self.print_expr_repeat(element, count, attrs)?; + self.print_expr_repeat(element, count, attrs); } ast::ExprKind::Struct(ref path, ref fields, ref wth) => { - self.print_expr_struct(path, &fields[..], wth, attrs)?; + self.print_expr_struct(path, &fields[..], wth, attrs); } ast::ExprKind::Tup(ref exprs) => { - self.print_expr_tup(&exprs[..], attrs)?; + self.print_expr_tup(&exprs[..], attrs); } ast::ExprKind::Call(ref func, ref args) => { - self.print_expr_call(func, &args[..])?; + self.print_expr_call(func, &args[..]); } ast::ExprKind::MethodCall(ref segment, ref args) => { - self.print_expr_method_call(segment, &args[..])?; + self.print_expr_method_call(segment, &args[..]); } ast::ExprKind::Binary(op, ref lhs, ref rhs) => { - self.print_expr_binary(op, lhs, rhs)?; + self.print_expr_binary(op, lhs, rhs); } ast::ExprKind::Unary(op, ref expr) => { - self.print_expr_unary(op, expr)?; + self.print_expr_unary(op, expr); } ast::ExprKind::AddrOf(m, ref expr) => { - self.print_expr_addr_of(m, expr)?; + self.print_expr_addr_of(m, expr); } ast::ExprKind::Lit(ref lit) => { - self.print_literal(lit)?; + self.print_literal(lit); } ast::ExprKind::Cast(ref expr, ref ty) => { let prec = AssocOp::As.precedence() as i8; - self.print_expr_maybe_paren(expr, prec)?; - self.s.space()?; - self.word_space("as")?; - self.print_type(ty)?; + self.print_expr_maybe_paren(expr, prec); + self.s.space(); + self.word_space("as"); + self.print_type(ty); } ast::ExprKind::Type(ref expr, ref ty) => { let prec = AssocOp::Colon.precedence() as i8; - self.print_expr_maybe_paren(expr, prec)?; - self.word_space(":")?; - self.print_type(ty)?; + self.print_expr_maybe_paren(expr, prec); + self.word_space(":"); + self.print_type(ty); } - ast::ExprKind::If(ref test, ref blk, ref elseopt) => { - self.print_if(test, blk, elseopt.as_ref().map(|e| &**e))?; + ast::ExprKind::Let(ref pats, ref scrutinee) => { + self.print_let(pats, scrutinee); } - ast::ExprKind::IfLet(ref pats, ref expr, ref blk, ref elseopt) => { - self.print_if_let(pats, expr, blk, elseopt.as_ref().map(|e| &**e))?; + ast::ExprKind::If(ref test, ref blk, ref elseopt) => { + self.print_if(test, blk, elseopt.as_ref().map(|e| &**e)); } ast::ExprKind::While(ref test, ref blk, opt_label) => { if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; + self.print_ident(label.ident); + self.word_space(":"); } - self.head("while")?; - self.print_expr_as_cond(test)?; - self.s.space()?; - self.print_block_with_attrs(blk, attrs)?; - } - ast::ExprKind::WhileLet(ref pats, ref expr, ref blk, opt_label) => { - if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; - } - self.head("while let")?; - self.print_pats(pats)?; - self.s.space()?; - self.word_space("=")?; - self.print_expr_as_cond(expr)?; - self.s.space()?; - self.print_block_with_attrs(blk, attrs)?; + self.head("while"); + self.print_expr_as_cond(test); + self.s.space(); + self.print_block_with_attrs(blk, attrs); } ast::ExprKind::ForLoop(ref pat, ref iter, ref blk, opt_label) => { if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; + self.print_ident(label.ident); + self.word_space(":"); } - self.head("for")?; - self.print_pat(pat)?; - self.s.space()?; - self.word_space("in")?; - self.print_expr_as_cond(iter)?; - self.s.space()?; - self.print_block_with_attrs(blk, attrs)?; + self.head("for"); + self.print_pat(pat); + self.s.space(); + self.word_space("in"); + self.print_expr_as_cond(iter); + self.s.space(); + self.print_block_with_attrs(blk, attrs); } ast::ExprKind::Loop(ref blk, opt_label) => { if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; + self.print_ident(label.ident); + self.word_space(":"); } - self.head("loop")?; - self.s.space()?; - self.print_block_with_attrs(blk, attrs)?; + self.head("loop"); + self.s.space(); + self.print_block_with_attrs(blk, attrs); } ast::ExprKind::Match(ref expr, ref arms) => { - self.cbox(INDENT_UNIT)?; - self.ibox(4)?; - self.word_nbsp("match")?; - self.print_expr_as_cond(expr)?; - self.s.space()?; - self.bopen()?; - self.print_inner_attributes_no_trailing_hardbreak(attrs)?; + self.cbox(INDENT_UNIT); + self.ibox(INDENT_UNIT); + self.word_nbsp("match"); + self.print_expr_as_cond(expr); + self.s.space(); + self.bopen(); + self.print_inner_attributes_no_trailing_hardbreak(attrs); for arm in arms { - self.print_arm(arm)?; + self.print_arm(arm); } - self.bclose_(expr.span, INDENT_UNIT)?; + self.bclose(expr.span); } ast::ExprKind::Closure( capture_clause, asyncness, movability, ref decl, ref body, _) => { - self.print_movability(movability)?; - self.print_asyncness(asyncness)?; - self.print_capture_clause(capture_clause)?; + self.print_movability(movability); + self.print_asyncness(asyncness); + self.print_capture_clause(capture_clause); - self.print_fn_block_args(decl)?; - self.s.space()?; - self.print_expr(body)?; - self.end()?; // need to close a box + self.print_fn_block_args(decl); + self.s.space(); + self.print_expr(body); + self.end(); // need to close a box // a box will be closed by print_expr, but we didn't want an overall // wrapper so we closed the corresponding opening. so create an // empty box to satisfy the close. - self.ibox(0)?; + self.ibox(0); } ast::ExprKind::Block(ref blk, opt_label) => { if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; + self.print_ident(label.ident); + self.word_space(":"); } // containing cbox, will be closed by print-block at } - self.cbox(INDENT_UNIT)?; + self.cbox(INDENT_UNIT); // head-box, will be closed by print-block after { - self.ibox(0)?; - self.print_block_with_attrs(blk, attrs)?; + self.ibox(0); + self.print_block_with_attrs(blk, attrs); } ast::ExprKind::Async(capture_clause, _, ref blk) => { - self.word_nbsp("async")?; - self.print_capture_clause(capture_clause)?; - self.s.space()?; + self.word_nbsp("async"); + self.print_capture_clause(capture_clause); + self.s.space(); // cbox/ibox in analogy to the `ExprKind::Block` arm above - self.cbox(INDENT_UNIT)?; - self.ibox(0)?; - self.print_block_with_attrs(blk, attrs)?; + self.cbox(INDENT_UNIT); + self.ibox(0); + self.print_block_with_attrs(blk, attrs); + } + ast::ExprKind::Await(ref expr) => { + self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX); + self.s.word(".await"); } ast::ExprKind::Assign(ref lhs, ref rhs) => { let prec = AssocOp::Assign.precedence() as i8; - self.print_expr_maybe_paren(lhs, prec + 1)?; - self.s.space()?; - self.word_space("=")?; - self.print_expr_maybe_paren(rhs, prec)?; + self.print_expr_maybe_paren(lhs, prec + 1); + self.s.space(); + self.word_space("="); + self.print_expr_maybe_paren(rhs, prec); } ast::ExprKind::AssignOp(op, ref lhs, ref rhs) => { let prec = AssocOp::Assign.precedence() as i8; - self.print_expr_maybe_paren(lhs, prec + 1)?; - self.s.space()?; - self.s.word(op.node.to_string())?; - self.word_space("=")?; - self.print_expr_maybe_paren(rhs, prec)?; + self.print_expr_maybe_paren(lhs, prec + 1); + self.s.space(); + self.s.word(op.node.to_string()); + self.word_space("="); + self.print_expr_maybe_paren(rhs, prec); } ast::ExprKind::Field(ref expr, ident) => { - self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX)?; - self.s.word(".")?; - self.print_ident(ident)?; + self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX); + self.s.word("."); + self.print_ident(ident); } ast::ExprKind::Index(ref expr, ref index) => { - self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX)?; - self.s.word("[")?; - self.print_expr(index)?; - self.s.word("]")?; + self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX); + self.s.word("["); + self.print_expr(index); + self.s.word("]"); } ast::ExprKind::Range(ref start, ref end, limits) => { // Special case for `Range`. `AssocOp` claims that `Range` has higher precedence @@ -2278,55 +2144,55 @@ impl<'a> State<'a> { // a "normal" binop gets parenthesized. (`LOr` is the lowest-precedence binop.) let fake_prec = AssocOp::LOr.precedence() as i8; if let Some(ref e) = *start { - self.print_expr_maybe_paren(e, fake_prec)?; + self.print_expr_maybe_paren(e, fake_prec); } if limits == ast::RangeLimits::HalfOpen { - self.s.word("..")?; + self.s.word(".."); } else { - self.s.word("..=")?; + self.s.word("..="); } if let Some(ref e) = *end { - self.print_expr_maybe_paren(e, fake_prec)?; + self.print_expr_maybe_paren(e, fake_prec); } } ast::ExprKind::Path(None, ref path) => { - self.print_path(path, true, 0)? + self.print_path(path, true, 0) } ast::ExprKind::Path(Some(ref qself), ref path) => { - self.print_qpath(path, qself, true)? + self.print_qpath(path, qself, true) } ast::ExprKind::Break(opt_label, ref opt_expr) => { - self.s.word("break")?; - self.s.space()?; + self.s.word("break"); + self.s.space(); if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.s.space()?; + self.print_ident(label.ident); + self.s.space(); } if let Some(ref expr) = *opt_expr { - self.print_expr_maybe_paren(expr, parser::PREC_JUMP)?; - self.s.space()?; + self.print_expr_maybe_paren(expr, parser::PREC_JUMP); + self.s.space(); } } ast::ExprKind::Continue(opt_label) => { - self.s.word("continue")?; - self.s.space()?; + self.s.word("continue"); + self.s.space(); if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.s.space()? + self.print_ident(label.ident); + self.s.space() } } ast::ExprKind::Ret(ref result) => { - self.s.word("return")?; + self.s.word("return"); if let Some(ref expr) = *result { - self.s.word(" ")?; - self.print_expr_maybe_paren(expr, parser::PREC_JUMP)?; + self.s.word(" "); + self.print_expr_maybe_paren(expr, parser::PREC_JUMP); } } ast::ExprKind::InlineAsm(ref a) => { - self.s.word("asm!")?; - self.popen()?; - self.print_string(&a.asm.as_str(), a.asm_str_style)?; - self.word_space(":")?; + self.s.word("asm!"); + self.popen(); + self.print_string(&a.asm.as_str(), a.asm_str_style); + self.word_space(":"); self.commasep(Inconsistent, &a.outputs, |s, out| { let constraint = out.constraint.as_str(); @@ -2334,33 +2200,30 @@ impl<'a> State<'a> { match ch.next() { Some('=') if out.is_rw => { s.print_string(&format!("+{}", ch.as_str()), - ast::StrStyle::Cooked)? + ast::StrStyle::Cooked) } - _ => s.print_string(&constraint, ast::StrStyle::Cooked)? + _ => s.print_string(&constraint, ast::StrStyle::Cooked) } - s.popen()?; - s.print_expr(&out.expr)?; - s.pclose()?; - Ok(()) - })?; - self.s.space()?; - self.word_space(":")?; + s.popen(); + s.print_expr(&out.expr); + s.pclose(); + }); + self.s.space(); + self.word_space(":"); self.commasep(Inconsistent, &a.inputs, |s, &(co, ref o)| { - s.print_string(&co.as_str(), ast::StrStyle::Cooked)?; - s.popen()?; - s.print_expr(o)?; - s.pclose()?; - Ok(()) - })?; - self.s.space()?; - self.word_space(":")?; + s.print_string(&co.as_str(), ast::StrStyle::Cooked); + s.popen(); + s.print_expr(o); + s.pclose(); + }); + self.s.space(); + self.word_space(":"); self.commasep(Inconsistent, &a.clobbers, |s, co| { - s.print_string(&co.as_str(), ast::StrStyle::Cooked)?; - Ok(()) - })?; + s.print_string(&co.as_str(), ast::StrStyle::Cooked); + }); let mut options = vec![]; if a.volatile { @@ -2374,608 +2237,436 @@ impl<'a> State<'a> { } if !options.is_empty() { - self.s.space()?; - self.word_space(":")?; + self.s.space(); + self.word_space(":"); self.commasep(Inconsistent, &options, |s, &co| { - s.print_string(co, ast::StrStyle::Cooked)?; - Ok(()) - })?; + s.print_string(co, ast::StrStyle::Cooked); + }); } - self.pclose()?; + self.pclose(); } - ast::ExprKind::Mac(ref m) => self.print_mac(m)?, + ast::ExprKind::Mac(ref m) => self.print_mac(m), ast::ExprKind::Paren(ref e) => { - self.popen()?; - self.print_inner_attributes_inline(attrs)?; - self.print_expr(e)?; - self.pclose()?; + self.popen(); + self.print_inner_attributes_inline(attrs); + self.print_expr(e); + self.pclose(); }, ast::ExprKind::Yield(ref e) => { - self.s.word("yield")?; + self.s.word("yield"); match *e { Some(ref expr) => { - self.s.space()?; - self.print_expr_maybe_paren(expr, parser::PREC_JUMP)?; + self.s.space(); + self.print_expr_maybe_paren(expr, parser::PREC_JUMP); } _ => () } } ast::ExprKind::Try(ref e) => { - self.print_expr_maybe_paren(e, parser::PREC_POSTFIX)?; - self.s.word("?")? + self.print_expr_maybe_paren(e, parser::PREC_POSTFIX); + self.s.word("?") } ast::ExprKind::TryBlock(ref blk) => { - self.head("try")?; - self.s.space()?; - self.print_block_with_attrs(blk, attrs)? + self.head("try"); + self.s.space(); + self.print_block_with_attrs(blk, attrs) } ast::ExprKind::Err => { - self.popen()?; - self.s.word("/*ERROR*/")?; - self.pclose()? + self.popen(); + self.s.word("/*ERROR*/"); + self.pclose() } } - self.ann.post(self, AnnNode::Expr(expr))?; - self.end() + self.ann.post(self, AnnNode::Expr(expr)); + self.end(); } - pub fn print_local_decl(&mut self, loc: &ast::Local) -> io::Result<()> { - self.print_pat(&loc.pat)?; + crate fn print_local_decl(&mut self, loc: &ast::Local) { + self.print_pat(&loc.pat); if let Some(ref ty) = loc.ty { - self.word_space(":")?; - self.print_type(ty)?; - } - Ok(()) - } - - pub fn print_ident(&mut self, ident: ast::Ident) -> io::Result<()> { - if ident.is_raw_guess() { - self.s.word(format!("r#{}", ident))?; - } else { - self.s.word(ident.as_str().get())?; + self.word_space(":"); + self.print_type(ty); } - self.ann.post(self, AnnNode::Ident(&ident)) } - pub fn print_usize(&mut self, i: usize) -> io::Result<()> { + crate fn print_usize(&mut self, i: usize) { self.s.word(i.to_string()) } - pub fn print_name(&mut self, name: ast::Name) -> io::Result<()> { - self.s.word(name.as_str().get())?; + crate fn print_name(&mut self, name: ast::Name) { + self.s.word(name.as_str().to_string()); self.ann.post(self, AnnNode::Name(&name)) } - pub fn print_for_decl(&mut self, loc: &ast::Local, - coll: &ast::Expr) -> io::Result<()> { - self.print_local_decl(loc)?; - self.s.space()?; - self.word_space("in")?; - self.print_expr(coll) - } - - fn print_path(&mut self, - path: &ast::Path, - colons_before_params: bool, - depth: usize) - -> io::Result<()> - { - self.maybe_print_comment(path.span.lo())?; - - for (i, segment) in path.segments[..path.segments.len() - depth].iter().enumerate() { - if i > 0 { - self.s.word("::")? - } - self.print_path_segment(segment, colons_before_params)?; - } - - Ok(()) - } - - fn print_path_segment(&mut self, - segment: &ast::PathSegment, - colons_before_params: bool) - -> io::Result<()> - { - if segment.ident.name != keywords::PathRoot.name() { - if segment.ident.name == keywords::DollarCrate.name() { - self.print_dollar_crate(segment.ident)?; - } else { - self.print_ident(segment.ident)?; - } - if let Some(ref args) = segment.args { - self.print_generic_args(args, colons_before_params)?; - } - } - Ok(()) - } - fn print_qpath(&mut self, path: &ast::Path, qself: &ast::QSelf, colons_before_params: bool) - -> io::Result<()> { - self.s.word("<")?; - self.print_type(&qself.ty)?; + self.s.word("<"); + self.print_type(&qself.ty); if qself.position > 0 { - self.s.space()?; - self.word_space("as")?; + self.s.space(); + self.word_space("as"); let depth = path.segments.len() - qself.position; - self.print_path(path, false, depth)?; + self.print_path(path, false, depth); } - self.s.word(">")?; - self.s.word("::")?; + self.s.word(">"); + self.s.word("::"); let item_segment = path.segments.last().unwrap(); - self.print_ident(item_segment.ident)?; + self.print_ident(item_segment.ident); match item_segment.args { Some(ref args) => self.print_generic_args(args, colons_before_params), - None => Ok(()), - } - } - - fn print_generic_args(&mut self, - args: &ast::GenericArgs, - colons_before_params: bool) - -> io::Result<()> - { - if colons_before_params { - self.s.word("::")? - } - - match *args { - ast::GenericArgs::AngleBracketed(ref data) => { - self.s.word("<")?; - - self.commasep(Inconsistent, &data.args, |s, generic_arg| { - s.print_generic_arg(generic_arg) - })?; - - let mut comma = data.args.len() != 0; - - for binding in data.bindings.iter() { - if comma { - self.word_space(",")? - } - self.print_ident(binding.ident)?; - self.s.space()?; - self.word_space("=")?; - self.print_type(&binding.ty)?; - comma = true; - } - - self.s.word(">")? - } - - ast::GenericArgs::Parenthesized(ref data) => { - self.s.word("(")?; - self.commasep( - Inconsistent, - &data.inputs, - |s, ty| s.print_type(ty))?; - self.s.word(")")?; - - if let Some(ref ty) = data.output { - self.space_if_not_bol()?; - self.word_space("->")?; - self.print_type(ty)?; - } - } + None => {}, } - - Ok(()) } - pub fn print_pat(&mut self, pat: &ast::Pat) -> io::Result<()> { - self.maybe_print_comment(pat.span.lo())?; - self.ann.pre(self, AnnNode::Pat(pat))?; + crate fn print_pat(&mut self, pat: &ast::Pat) { + self.maybe_print_comment(pat.span.lo()); + self.ann.pre(self, AnnNode::Pat(pat)); /* Pat isn't normalized, but the beauty of it is that it doesn't matter */ match pat.node { - PatKind::Wild => self.s.word("_")?, + PatKind::Wild => self.s.word("_"), PatKind::Ident(binding_mode, ident, ref sub) => { match binding_mode { ast::BindingMode::ByRef(mutbl) => { - self.word_nbsp("ref")?; - self.print_mutability(mutbl)?; + self.word_nbsp("ref"); + self.print_mutability(mutbl); } ast::BindingMode::ByValue(ast::Mutability::Immutable) => {} ast::BindingMode::ByValue(ast::Mutability::Mutable) => { - self.word_nbsp("mut")?; + self.word_nbsp("mut"); } } - self.print_ident(ident)?; + self.print_ident(ident); if let Some(ref p) = *sub { - self.s.word("@")?; - self.print_pat(p)?; + self.s.word("@"); + self.print_pat(p); } } - PatKind::TupleStruct(ref path, ref elts, ddpos) => { - self.print_path(path, true, 0)?; - self.popen()?; - if let Some(ddpos) = ddpos { - self.commasep(Inconsistent, &elts[..ddpos], |s, p| s.print_pat(p))?; - if ddpos != 0 { - self.word_space(",")?; - } - self.s.word("..")?; - if ddpos != elts.len() { - self.s.word(",")?; - self.commasep(Inconsistent, &elts[ddpos..], |s, p| s.print_pat(p))?; - } - } else { - self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p))?; - } - self.pclose()?; + PatKind::TupleStruct(ref path, ref elts) => { + self.print_path(path, true, 0); + self.popen(); + self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p)); + self.pclose(); } PatKind::Path(None, ref path) => { - self.print_path(path, true, 0)?; + self.print_path(path, true, 0); } PatKind::Path(Some(ref qself), ref path) => { - self.print_qpath(path, qself, false)?; + self.print_qpath(path, qself, false); } PatKind::Struct(ref path, ref fields, etc) => { - self.print_path(path, true, 0)?; - self.nbsp()?; - self.word_space("{")?; + self.print_path(path, true, 0); + self.nbsp(); + self.word_space("{"); self.commasep_cmnt( Consistent, &fields[..], |s, f| { - s.cbox(INDENT_UNIT)?; + s.cbox(INDENT_UNIT); if !f.node.is_shorthand { - s.print_ident(f.node.ident)?; - s.word_nbsp(":")?; + s.print_ident(f.node.ident); + s.word_nbsp(":"); } - s.print_pat(&f.node.pat)?; - s.end() + s.print_pat(&f.node.pat); + s.end(); }, - |f| f.node.pat.span)?; + |f| f.node.pat.span); if etc { - if !fields.is_empty() { self.word_space(",")?; } - self.s.word("..")?; - } - self.s.space()?; - self.s.word("}")?; - } - PatKind::Tuple(ref elts, ddpos) => { - self.popen()?; - if let Some(ddpos) = ddpos { - self.commasep(Inconsistent, &elts[..ddpos], |s, p| s.print_pat(p))?; - if ddpos != 0 { - self.word_space(",")?; - } - self.s.word("..")?; - if ddpos != elts.len() { - self.s.word(",")?; - self.commasep(Inconsistent, &elts[ddpos..], |s, p| s.print_pat(p))?; - } - } else { - self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p))?; - if elts.len() == 1 { - self.s.word(",")?; - } + if !fields.is_empty() { self.word_space(","); } + self.s.word(".."); + } + self.s.space(); + self.s.word("}"); + } + PatKind::Tuple(ref elts) => { + self.popen(); + self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p)); + if elts.len() == 1 { + self.s.word(","); } - self.pclose()?; + self.pclose(); } PatKind::Box(ref inner) => { - self.s.word("box ")?; - self.print_pat(inner)?; + self.s.word("box "); + self.print_pat(inner); } PatKind::Ref(ref inner, mutbl) => { - self.s.word("&")?; + self.s.word("&"); if mutbl == ast::Mutability::Mutable { - self.s.word("mut ")?; + self.s.word("mut "); } - self.print_pat(inner)?; + self.print_pat(inner); } - PatKind::Lit(ref e) => self.print_expr(&**e)?, + PatKind::Lit(ref e) => self.print_expr(&**e), PatKind::Range(ref begin, ref end, Spanned { node: ref end_kind, .. }) => { - self.print_expr(begin)?; - self.s.space()?; + self.print_expr(begin); + self.s.space(); match *end_kind { - RangeEnd::Included(RangeSyntax::DotDotDot) => self.s.word("...")?, - RangeEnd::Included(RangeSyntax::DotDotEq) => self.s.word("..=")?, - RangeEnd::Excluded => self.s.word("..")?, - } - self.print_expr(end)?; - } - PatKind::Slice(ref before, ref slice, ref after) => { - self.s.word("[")?; - self.commasep(Inconsistent, - &before[..], - |s, p| s.print_pat(p))?; - if let Some(ref p) = *slice { - if !before.is_empty() { self.word_space(",")?; } - if let PatKind::Wild = p.node { - // Print nothing - } else { - self.print_pat(p)?; - } - self.s.word("..")?; - if !after.is_empty() { self.word_space(",")?; } + RangeEnd::Included(RangeSyntax::DotDotDot) => self.s.word("..."), + RangeEnd::Included(RangeSyntax::DotDotEq) => self.s.word("..="), + RangeEnd::Excluded => self.s.word(".."), } - self.commasep(Inconsistent, - &after[..], - |s, p| s.print_pat(p))?; - self.s.word("]")?; + self.print_expr(end); } + PatKind::Slice(ref elts) => { + self.s.word("["); + self.commasep(Inconsistent, &elts[..], |s, p| s.print_pat(p)); + self.s.word("]"); + } + PatKind::Rest => self.s.word(".."), PatKind::Paren(ref inner) => { - self.popen()?; - self.print_pat(inner)?; - self.pclose()?; + self.popen(); + self.print_pat(inner); + self.pclose(); } - PatKind::Mac(ref m) => self.print_mac(m)?, + PatKind::Mac(ref m) => self.print_mac(m), } self.ann.post(self, AnnNode::Pat(pat)) } - fn print_pats(&mut self, pats: &[P<ast::Pat>]) -> io::Result<()> { + fn print_pats(&mut self, pats: &[P<ast::Pat>]) { let mut first = true; for p in pats { if first { first = false; } else { - self.s.space()?; - self.word_space("|")?; + self.s.space(); + self.word_space("|"); } - self.print_pat(p)?; + self.print_pat(p); } - Ok(()) } - fn print_arm(&mut self, arm: &ast::Arm) -> io::Result<()> { + fn print_arm(&mut self, arm: &ast::Arm) { // I have no idea why this check is necessary, but here it // is :( if arm.attrs.is_empty() { - self.s.space()?; + self.s.space(); } - self.cbox(INDENT_UNIT)?; - self.ibox(0)?; - self.maybe_print_comment(arm.pats[0].span.lo())?; - self.print_outer_attributes(&arm.attrs)?; - self.print_pats(&arm.pats)?; - self.s.space()?; - if let Some(ref g) = arm.guard { - match g { - ast::Guard::If(ref e) => { - self.word_space("if")?; - self.print_expr(e)?; - self.s.space()?; - } - } + self.cbox(INDENT_UNIT); + self.ibox(0); + self.maybe_print_comment(arm.pats[0].span.lo()); + self.print_outer_attributes(&arm.attrs); + self.print_pats(&arm.pats); + self.s.space(); + if let Some(ref e) = arm.guard { + self.word_space("if"); + self.print_expr(e); + self.s.space(); } - self.word_space("=>")?; + self.word_space("=>"); match arm.body.node { ast::ExprKind::Block(ref blk, opt_label) => { if let Some(label) = opt_label { - self.print_ident(label.ident)?; - self.word_space(":")?; + self.print_ident(label.ident); + self.word_space(":"); } // the block will close the pattern's ibox - self.print_block_unclosed_indent(blk, INDENT_UNIT)?; + self.print_block_unclosed_indent(blk); // If it is a user-provided unsafe block, print a comma after it if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules { - self.s.word(",")?; + self.s.word(","); } } _ => { - self.end()?; // close the ibox for the pattern - self.print_expr(&arm.body)?; - self.s.word(",")?; + self.end(); // close the ibox for the pattern + self.print_expr(&arm.body); + self.s.word(","); } } - self.end() // close enclosing cbox + self.end(); // close enclosing cbox } - fn print_explicit_self(&mut self, explicit_self: &ast::ExplicitSelf) -> io::Result<()> { + fn print_explicit_self(&mut self, explicit_self: &ast::ExplicitSelf) { match explicit_self.node { SelfKind::Value(m) => { - self.print_mutability(m)?; + self.print_mutability(m); self.s.word("self") } SelfKind::Region(ref lt, m) => { - self.s.word("&")?; - self.print_opt_lifetime(lt)?; - self.print_mutability(m)?; + self.s.word("&"); + self.print_opt_lifetime(lt); + self.print_mutability(m); self.s.word("self") } SelfKind::Explicit(ref typ, m) => { - self.print_mutability(m)?; - self.s.word("self")?; - self.word_space(":")?; + self.print_mutability(m); + self.s.word("self"); + self.word_space(":"); self.print_type(typ) } } } - pub fn print_fn(&mut self, + crate fn print_fn(&mut self, decl: &ast::FnDecl, header: ast::FnHeader, name: Option<ast::Ident>, generics: &ast::Generics, - vis: &ast::Visibility) -> io::Result<()> { - self.print_fn_header_info(header, vis)?; + vis: &ast::Visibility) { + self.print_fn_header_info(header, vis); if let Some(name) = name { - self.nbsp()?; - self.print_ident(name)?; + self.nbsp(); + self.print_ident(name); } - self.print_generic_params(&generics.params)?; - self.print_fn_args_and_ret(decl)?; + self.print_generic_params(&generics.params); + self.print_fn_args_and_ret(decl); self.print_where_clause(&generics.where_clause) } - pub fn print_fn_args_and_ret(&mut self, decl: &ast::FnDecl) - -> io::Result<()> { - self.popen()?; - self.commasep(Inconsistent, &decl.inputs, |s, arg| s.print_arg(arg, false))?; - self.pclose()?; + crate fn print_fn_args_and_ret(&mut self, decl: &ast::FnDecl) { + self.popen(); + self.commasep(Inconsistent, &decl.inputs, |s, arg| s.print_arg(arg, false)); + self.pclose(); self.print_fn_output(decl) } - pub fn print_fn_block_args( - &mut self, - decl: &ast::FnDecl) - -> io::Result<()> { - self.s.word("|")?; - self.commasep(Inconsistent, &decl.inputs, |s, arg| s.print_arg(arg, true))?; - self.s.word("|")?; + crate fn print_fn_block_args(&mut self, decl: &ast::FnDecl) { + self.s.word("|"); + self.commasep(Inconsistent, &decl.inputs, |s, arg| s.print_arg(arg, true)); + self.s.word("|"); if let ast::FunctionRetTy::Default(..) = decl.output { - return Ok(()); + return; } - self.space_if_not_bol()?; - self.word_space("->")?; + self.space_if_not_bol(); + self.word_space("->"); match decl.output { ast::FunctionRetTy::Ty(ref ty) => { - self.print_type(ty)?; + self.print_type(ty); self.maybe_print_comment(ty.span.lo()) } ast::FunctionRetTy::Default(..) => unreachable!(), } } - pub fn print_movability(&mut self, movability: ast::Movability) - -> io::Result<()> { + crate fn print_movability(&mut self, movability: ast::Movability) { match movability { ast::Movability::Static => self.word_space("static"), - ast::Movability::Movable => Ok(()), + ast::Movability::Movable => {}, } } - pub fn print_asyncness(&mut self, asyncness: ast::IsAsync) - -> io::Result<()> { + crate fn print_asyncness(&mut self, asyncness: ast::IsAsync) { if asyncness.is_async() { - self.word_nbsp("async")?; + self.word_nbsp("async"); } - Ok(()) } - pub fn print_capture_clause(&mut self, capture_clause: ast::CaptureBy) - -> io::Result<()> { + crate fn print_capture_clause(&mut self, capture_clause: ast::CaptureBy) { match capture_clause { ast::CaptureBy::Value => self.word_space("move"), - ast::CaptureBy::Ref => Ok(()), + ast::CaptureBy::Ref => {}, } } - pub fn print_type_bounds(&mut self, prefix: &'static str, bounds: &[ast::GenericBound]) - -> io::Result<()> { + crate fn print_type_bounds(&mut self, prefix: &'static str, bounds: &[ast::GenericBound]) { if !bounds.is_empty() { - self.s.word(prefix)?; + self.s.word(prefix); let mut first = true; for bound in bounds { if !(first && prefix.is_empty()) { - self.nbsp()?; + self.nbsp(); } if first { first = false; } else { - self.word_space("+")?; + self.word_space("+"); } match bound { GenericBound::Trait(tref, modifier) => { if modifier == &TraitBoundModifier::Maybe { - self.s.word("?")?; + self.s.word("?"); } - self.print_poly_trait_ref(tref)?; + self.print_poly_trait_ref(tref); } - GenericBound::Outlives(lt) => self.print_lifetime(*lt)?, + GenericBound::Outlives(lt) => self.print_lifetime(*lt), } } } - Ok(()) } - pub fn print_lifetime(&mut self, lifetime: ast::Lifetime) -> io::Result<()> { + crate fn print_lifetime(&mut self, lifetime: ast::Lifetime) { self.print_name(lifetime.ident.name) } - pub fn print_lifetime_bounds(&mut self, lifetime: ast::Lifetime, bounds: &ast::GenericBounds) - -> io::Result<()> - { - self.print_lifetime(lifetime)?; + crate fn print_lifetime_bounds( + &mut self, lifetime: ast::Lifetime, bounds: &ast::GenericBounds) { + self.print_lifetime(lifetime); if !bounds.is_empty() { - self.s.word(": ")?; + self.s.word(": "); for (i, bound) in bounds.iter().enumerate() { if i != 0 { - self.s.word(" + ")?; + self.s.word(" + "); } match bound { - ast::GenericBound::Outlives(lt) => self.print_lifetime(*lt)?, + ast::GenericBound::Outlives(lt) => self.print_lifetime(*lt), _ => panic!(), } } } - Ok(()) } - pub fn print_generic_params( - &mut self, - generic_params: &[ast::GenericParam] - ) -> io::Result<()> { + crate fn print_generic_params(&mut self, generic_params: &[ast::GenericParam]) { if generic_params.is_empty() { - return Ok(()); + return; } - self.s.word("<")?; + self.s.word("<"); self.commasep(Inconsistent, &generic_params, |s, param| { + s.print_outer_attributes_inline(¶m.attrs); + match param.kind { ast::GenericParamKind::Lifetime => { - s.print_outer_attributes_inline(¶m.attrs)?; let lt = ast::Lifetime { id: param.id, ident: param.ident }; s.print_lifetime_bounds(lt, ¶m.bounds) } ast::GenericParamKind::Type { ref default } => { - s.print_outer_attributes_inline(¶m.attrs)?; - s.print_ident(param.ident)?; - s.print_type_bounds(":", ¶m.bounds)?; - match default { - Some(ref default) => { - s.s.space()?; - s.word_space("=")?; - s.print_type(default) - } - _ => Ok(()) + s.print_ident(param.ident); + s.print_type_bounds(":", ¶m.bounds); + if let Some(ref default) = default { + s.s.space(); + s.word_space("="); + s.print_type(default) } } ast::GenericParamKind::Const { ref ty } => { - s.print_outer_attributes_inline(¶m.attrs)?; - s.word_space("const")?; - s.print_ident(param.ident)?; - s.s.space()?; - s.word_space(":")?; - s.print_type(ty)?; + s.word_space("const"); + s.print_ident(param.ident); + s.s.space(); + s.word_space(":"); + s.print_type(ty); s.print_type_bounds(":", ¶m.bounds) } } - })?; + }); - self.s.word(">")?; - Ok(()) + self.s.word(">"); } - pub fn print_where_clause(&mut self, where_clause: &ast::WhereClause) - -> io::Result<()> { + crate fn print_where_clause(&mut self, where_clause: &ast::WhereClause) { if where_clause.predicates.is_empty() { - return Ok(()) + return; } - self.s.space()?; - self.word_space("where")?; + self.s.space(); + self.word_space("where"); for (i, predicate) in where_clause.predicates.iter().enumerate() { if i != 0 { - self.word_space(",")?; + self.word_space(","); } match *predicate { @@ -2985,138 +2676,135 @@ impl<'a> State<'a> { ref bounds, .. }) => { - self.print_formal_generic_params(bound_generic_params)?; - self.print_type(bounded_ty)?; - self.print_type_bounds(":", bounds)?; + self.print_formal_generic_params(bound_generic_params); + self.print_type(bounded_ty); + self.print_type_bounds(":", bounds); } ast::WherePredicate::RegionPredicate(ast::WhereRegionPredicate{ref lifetime, ref bounds, ..}) => { - self.print_lifetime_bounds(*lifetime, bounds)?; + self.print_lifetime_bounds(*lifetime, bounds); } ast::WherePredicate::EqPredicate(ast::WhereEqPredicate{ref lhs_ty, ref rhs_ty, ..}) => { - self.print_type(lhs_ty)?; - self.s.space()?; - self.word_space("=")?; - self.print_type(rhs_ty)?; + self.print_type(lhs_ty); + self.s.space(); + self.word_space("="); + self.print_type(rhs_ty); } } } - - Ok(()) } - pub fn print_use_tree(&mut self, tree: &ast::UseTree) -> io::Result<()> { + crate fn print_use_tree(&mut self, tree: &ast::UseTree) { match tree.kind { ast::UseTreeKind::Simple(rename, ..) => { - self.print_path(&tree.prefix, false, 0)?; + self.print_path(&tree.prefix, false, 0); if let Some(rename) = rename { - self.s.space()?; - self.word_space("as")?; - self.print_ident(rename)?; + self.s.space(); + self.word_space("as"); + self.print_ident(rename); } } ast::UseTreeKind::Glob => { if !tree.prefix.segments.is_empty() { - self.print_path(&tree.prefix, false, 0)?; - self.s.word("::")?; + self.print_path(&tree.prefix, false, 0); + self.s.word("::"); } - self.s.word("*")?; + self.s.word("*"); } ast::UseTreeKind::Nested(ref items) => { if tree.prefix.segments.is_empty() { - self.s.word("{")?; + self.s.word("{"); } else { - self.print_path(&tree.prefix, false, 0)?; - self.s.word("::{")?; + self.print_path(&tree.prefix, false, 0); + self.s.word("::{"); } self.commasep(Inconsistent, &items[..], |this, &(ref tree, _)| { this.print_use_tree(tree) - })?; - self.s.word("}")?; + }); + self.s.word("}"); } } - - Ok(()) } - pub fn print_mutability(&mut self, - mutbl: ast::Mutability) -> io::Result<()> { + crate fn print_mutability(&mut self, mutbl: ast::Mutability) { match mutbl { ast::Mutability::Mutable => self.word_nbsp("mut"), - ast::Mutability::Immutable => Ok(()), + ast::Mutability::Immutable => {}, } } - pub fn print_mt(&mut self, mt: &ast::MutTy) -> io::Result<()> { - self.print_mutability(mt.mutbl)?; + crate fn print_mt(&mut self, mt: &ast::MutTy) { + self.print_mutability(mt.mutbl); self.print_type(&mt.ty) } - pub fn print_arg(&mut self, input: &ast::Arg, is_closure: bool) -> io::Result<()> { - self.ibox(INDENT_UNIT)?; + crate fn print_arg(&mut self, input: &ast::Arg, is_closure: bool) { + self.ibox(INDENT_UNIT); + + self.print_outer_attributes_inline(&input.attrs); + match input.ty.node { - ast::TyKind::Infer if is_closure => self.print_pat(&input.pat)?, + ast::TyKind::Infer if is_closure => self.print_pat(&input.pat), _ => { if let Some(eself) = input.to_self() { - self.print_explicit_self(&eself)?; + self.print_explicit_self(&eself); } else { let invalid = if let PatKind::Ident(_, ident, _) = input.pat.node { - ident.name == keywords::Invalid.name() + ident.name == kw::Invalid } else { false }; if !invalid { - self.print_pat(&input.pat)?; - self.s.word(":")?; - self.s.space()?; + self.print_pat(&input.pat); + self.s.word(":"); + self.s.space(); } - self.print_type(&input.ty)?; + self.print_type(&input.ty); } } } - self.end() + self.end(); } - pub fn print_fn_output(&mut self, decl: &ast::FnDecl) -> io::Result<()> { + crate fn print_fn_output(&mut self, decl: &ast::FnDecl) { if let ast::FunctionRetTy::Default(..) = decl.output { - return Ok(()); + return; } - self.space_if_not_bol()?; - self.ibox(INDENT_UNIT)?; - self.word_space("->")?; + self.space_if_not_bol(); + self.ibox(INDENT_UNIT); + self.word_space("->"); match decl.output { ast::FunctionRetTy::Default(..) => unreachable!(), ast::FunctionRetTy::Ty(ref ty) => - self.print_type(ty)? + self.print_type(ty), } - self.end()?; + self.end(); match decl.output { ast::FunctionRetTy::Ty(ref output) => self.maybe_print_comment(output.span.lo()), - _ => Ok(()) + _ => {} } } - pub fn print_ty_fn(&mut self, + crate fn print_ty_fn(&mut self, abi: abi::Abi, unsafety: ast::Unsafety, decl: &ast::FnDecl, name: Option<ast::Ident>, generic_params: &[ast::GenericParam]) - -> io::Result<()> { - self.ibox(INDENT_UNIT)?; + { + self.ibox(INDENT_UNIT); if !generic_params.is_empty() { - self.s.word("for")?; - self.print_generic_params(generic_params)?; + self.s.word("for"); + self.print_generic_params(generic_params); } let generics = ast::Generics { params: Vec::new(), where_clause: ast::WhereClause { - id: ast::DUMMY_NODE_ID, predicates: Vec::new(), span: syntax_pos::DUMMY_SP, }, @@ -3126,153 +2814,63 @@ impl<'a> State<'a> { ast::FnHeader { unsafety, abi, ..ast::FnHeader::default() }, name, &generics, - &source_map::dummy_spanned(ast::VisibilityKind::Inherited))?; - self.end() + &source_map::dummy_spanned(ast::VisibilityKind::Inherited)); + self.end(); } - pub fn maybe_print_trailing_comment(&mut self, span: syntax_pos::Span, + crate fn maybe_print_trailing_comment(&mut self, span: syntax_pos::Span, next_pos: Option<BytePos>) - -> io::Result<()> { - let cm = match self.cm { - Some(cm) => cm, - _ => return Ok(()) - }; - if let Some(ref cmnt) = self.next_comment() { - if cmnt.style != comments::Trailing { return Ok(()) } - let span_line = cm.lookup_char_pos(span.hi()); - let comment_line = cm.lookup_char_pos(cmnt.pos); - let next = next_pos.unwrap_or_else(|| cmnt.pos + BytePos(1)); - if span.hi() < cmnt.pos && cmnt.pos < next && span_line.line == comment_line.line { - self.print_comment(cmnt)?; + { + if let Some(cmnts) = self.comments() { + if let Some(cmnt) = cmnts.trailing_comment(span, next_pos) { + self.print_comment(&cmnt); } } - Ok(()) } - pub fn print_remaining_comments(&mut self) -> io::Result<()> { + crate fn print_remaining_comments(&mut self) { // If there aren't any remaining comments, then we need to manually // make sure there is a line break at the end. if self.next_comment().is_none() { - self.s.hardbreak()?; + self.s.hardbreak(); } while let Some(ref cmnt) = self.next_comment() { - self.print_comment(cmnt)?; - } - Ok(()) - } - - pub fn print_opt_abi_and_extern_if_nondefault(&mut self, - opt_abi: Option<Abi>) - -> io::Result<()> { - match opt_abi { - Some(Abi::Rust) => Ok(()), - Some(abi) => { - self.word_nbsp("extern")?; - self.word_nbsp(abi.to_string()) - } - None => Ok(()) - } - } - - pub fn print_extern_opt_abi(&mut self, - opt_abi: Option<Abi>) -> io::Result<()> { - match opt_abi { - Some(abi) => { - self.word_nbsp("extern")?; - self.word_nbsp(abi.to_string()) - } - None => Ok(()) + self.print_comment(cmnt); } } - pub fn print_fn_header_info(&mut self, + crate fn print_fn_header_info(&mut self, header: ast::FnHeader, - vis: &ast::Visibility) -> io::Result<()> { - self.s.word(visibility_qualified(vis, ""))?; + vis: &ast::Visibility) { + self.s.word(visibility_qualified(vis, "")); match header.constness.node { ast::Constness::NotConst => {} - ast::Constness::Const => self.word_nbsp("const")? + ast::Constness::Const => self.word_nbsp("const") } - self.print_asyncness(header.asyncness.node)?; - self.print_unsafety(header.unsafety)?; + self.print_asyncness(header.asyncness.node); + self.print_unsafety(header.unsafety); if header.abi != Abi::Rust { - self.word_nbsp("extern")?; - self.word_nbsp(header.abi.to_string())?; + self.word_nbsp("extern"); + self.word_nbsp(header.abi.to_string()); } self.s.word("fn") } - pub fn print_unsafety(&mut self, s: ast::Unsafety) -> io::Result<()> { + crate fn print_unsafety(&mut self, s: ast::Unsafety) { match s { - ast::Unsafety::Normal => Ok(()), + ast::Unsafety::Normal => {}, ast::Unsafety::Unsafe => self.word_nbsp("unsafe"), } } - pub fn print_is_auto(&mut self, s: ast::IsAuto) -> io::Result<()> { + crate fn print_is_auto(&mut self, s: ast::IsAuto) { match s { ast::IsAuto::Yes => self.word_nbsp("auto"), - ast::IsAuto::No => Ok(()), + ast::IsAuto::No => {} } } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::ast; - use crate::source_map; - use crate::with_globals; - use syntax_pos; - - #[test] - fn test_fun_to_string() { - with_globals(|| { - let abba_ident = ast::Ident::from_str("abba"); - - let decl = ast::FnDecl { - inputs: Vec::new(), - output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP), - c_variadic: false - }; - let generics = ast::Generics::default(); - assert_eq!( - fun_to_string( - &decl, - ast::FnHeader { - unsafety: ast::Unsafety::Normal, - constness: source_map::dummy_spanned(ast::Constness::NotConst), - asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync), - abi: Abi::Rust, - }, - abba_ident, - &generics - ), - "fn abba()" - ); - }) - } - - #[test] - fn test_variant_to_string() { - with_globals(|| { - let ident = ast::Ident::from_str("principal_skinner"); - - let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ { - ident, - attrs: Vec::new(), - // making this up as I go.... ? - data: ast::VariantData::Unit(ast::DUMMY_NODE_ID), - disr_expr: None, - }); - - let varstr = variant_to_string(&var); - assert_eq!(varstr, "principal_skinner"); - }) - } -} diff --git a/src/libsyntax/print/pprust/tests.rs b/src/libsyntax/print/pprust/tests.rs new file mode 100644 index 00000000000..082a430e0ed --- /dev/null +++ b/src/libsyntax/print/pprust/tests.rs @@ -0,0 +1,69 @@ +use super::*; + +use crate::ast; +use crate::source_map; +use crate::with_default_globals; +use syntax_pos; + +fn fun_to_string( + decl: &ast::FnDecl, header: ast::FnHeader, name: ast::Ident, generics: &ast::Generics +) -> String { + to_string(|s| { + s.head(""); + s.print_fn(decl, header, Some(name), + generics, &source_map::dummy_spanned(ast::VisibilityKind::Inherited)); + s.end(); // Close the head box + s.end(); // Close the outer box + }) +} + +fn variant_to_string(var: &ast::Variant) -> String { + to_string(|s| s.print_variant(var)) +} + +#[test] +fn test_fun_to_string() { + with_default_globals(|| { + let abba_ident = ast::Ident::from_str("abba"); + + let decl = ast::FnDecl { + inputs: Vec::new(), + output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP), + c_variadic: false + }; + let generics = ast::Generics::default(); + assert_eq!( + fun_to_string( + &decl, + ast::FnHeader { + unsafety: ast::Unsafety::Normal, + constness: source_map::dummy_spanned(ast::Constness::NotConst), + asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync), + abi: Abi::Rust, + }, + abba_ident, + &generics + ), + "fn abba()" + ); + }) +} + +#[test] +fn test_variant_to_string() { + with_default_globals(|| { + let ident = ast::Ident::from_str("principal_skinner"); + + let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ { + ident, + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + // making this up as I go.... ? + data: ast::VariantData::Unit(ast::DUMMY_NODE_ID), + disr_expr: None, + }); + + let varstr = variant_to_string(&var); + assert_eq!(varstr, "principal_skinner"); + }) +} diff --git a/src/libsyntax/ptr.rs b/src/libsyntax/ptr.rs index 9afcb7c4621..b5eb8ca94c0 100644 --- a/src/libsyntax/ptr.rs +++ b/src/libsyntax/ptr.rs @@ -31,7 +31,7 @@ use std::iter::FromIterator; use std::ops::{Deref, DerefMut}; use std::{slice, vec}; -use serialize::{Encodable, Decodable, Encoder, Decoder}; +use rustc_serialize::{Encodable, Decodable, Encoder, Decoder}; use rustc_data_structures::stable_hasher::{StableHasher, StableHasherResult, HashStable}; @@ -41,11 +41,11 @@ pub struct P<T: ?Sized> { ptr: Box<T> } -#[allow(non_snake_case)] /// Construct a `P<T>` from a `T` value. +#[allow(non_snake_case)] pub fn P<T: 'static>(value: T) -> P<T> { P { - ptr: Box::new(value) + ptr: box value } } @@ -57,7 +57,8 @@ impl<T: 'static> P<T> { { f(*self.ptr) } - /// Equivalent to and_then(|x| x) + + /// Equivalent to `and_then(|x| x)`. pub fn into_inner(self) -> T { *self.ptr } @@ -132,8 +133,15 @@ impl<T: Encodable> Encodable for P<T> { } impl<T> P<[T]> { - pub fn new() -> P<[T]> { - P { ptr: Default::default() } + pub const fn new() -> P<[T]> { + // HACK(eddyb) bypass the lack of a `const fn` to create an empty `Box<[T]>` + // (as trait methods, `default` in this case, can't be `const fn` yet). + P { + ptr: unsafe { + use std::ptr::NonNull; + std::mem::transmute(NonNull::<[T; 0]>::dangling() as NonNull<[T]>) + }, + } } #[inline(never)] diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs index 62a6972122a..4e29c77c89e 100644 --- a/src/libsyntax/source_map.rs +++ b/src/libsyntax/source_map.rs @@ -7,10 +7,8 @@ //! within the SourceMap, which upon request can be converted to line and column //! information, source code snippets, etc. - pub use syntax_pos::*; -pub use syntax_pos::hygiene::{ExpnFormat, ExpnInfo}; -pub use ExpnFormat::*; +pub use syntax_pos::hygiene::{ExpnKind, ExpnInfo}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::StableHasher; @@ -26,12 +24,15 @@ use log::debug; use errors::SourceMapper; +#[cfg(test)] +mod tests; + /// Returns the span itself if it doesn't come from a macro expansion, /// otherwise return the call site span up to the `enclosing_sp` by /// following the `expn_info` chain. pub fn original_sp(sp: Span, enclosing_sp: Span) -> Span { - let call_site1 = sp.ctxt().outer().expn_info().map(|ei| ei.call_site); - let call_site2 = enclosing_sp.ctxt().outer().expn_info().map(|ei| ei.call_site); + let call_site1 = sp.ctxt().outer_expn_info().map(|ei| ei.call_site); + let call_site2 = enclosing_sp.ctxt().outer_expn_info().map(|ei| ei.call_site); match (call_site1, call_site2) { (None, _) => sp, (Some(call_site1), Some(call_site2)) if call_site1 == call_site2 => sp, @@ -124,12 +125,12 @@ impl StableSourceFileId { #[derive(Default)] pub(super) struct SourceMapFiles { - pub(super) source_files: Vec<Lrc<SourceFile>>, + source_files: Vec<Lrc<SourceFile>>, stable_id_to_source_file: FxHashMap<StableSourceFileId, Lrc<SourceFile>> } pub struct SourceMap { - pub(super) files: Lock<SourceMapFiles>, + files: Lock<SourceMapFiles>, file_loader: Box<dyn FileLoader + Sync + Send>, // This is used to apply the file path remapping as specified via // --remap-path-prefix to all SourceFiles allocated within this SourceMap. @@ -150,7 +151,7 @@ impl SourceMap { -> SourceMap { SourceMap { files: Default::default(), - file_loader: file_loader, + file_loader, path_mapping, } } @@ -191,6 +192,18 @@ impl SourceMap { /// If a file already exists in the source_map with the same id, that file is returned /// unmodified pub fn new_source_file(&self, filename: FileName, src: String) -> Lrc<SourceFile> { + self.try_new_source_file(filename, src) + .unwrap_or_else(|OffsetOverflowError| { + eprintln!("fatal error: rustc does not support files larger than 4GB"); + errors::FatalError.raise() + }) + } + + fn try_new_source_file( + &self, + filename: FileName, + src: String + ) -> Result<Lrc<SourceFile>, OffsetOverflowError> { let start_pos = self.next_start_pos(); // The path is used to determine the directory for loading submodules and @@ -212,7 +225,7 @@ impl SourceMap { was_remapped, Some(&unmapped_path)); - return match self.source_file_by_stable_id(file_id) { + let lrc_sf = match self.source_file_by_stable_id(file_id) { Some(lrc_sf) => lrc_sf, None => { let source_file = Lrc::new(SourceFile::new( @@ -221,7 +234,7 @@ impl SourceMap { unmapped_path, src, Pos::from_usize(start_pos), - )); + )?); let mut files = self.files.borrow_mut(); @@ -230,7 +243,8 @@ impl SourceMap { source_file } - } + }; + Ok(lrc_sf) } /// Allocates a new SourceFile representing a source file from an external @@ -383,21 +397,11 @@ impl SourceMap { let f = (*self.files.borrow().source_files)[idx].clone(); match f.lookup_line(pos) { - Some(line) => Ok(SourceFileAndLine { sf: f, line: line }), + Some(line) => Ok(SourceFileAndLine { sf: f, line }), None => Err(f) } } - pub fn lookup_char_pos_adj(&self, pos: BytePos) -> LocWithOpt { - let loc = self.lookup_char_pos(pos); - LocWithOpt { - filename: loc.file.name.clone(), - line: loc.line, - col: loc.col, - file: Some(loc.file) - } - } - /// Returns `Some(span)`, a union of the lhs and rhs span. The lhs must precede the rhs. If /// there are gaps between lhs and rhs, the resulting union will cross these gaps. /// For this to work, the spans have to be: @@ -438,10 +442,10 @@ impl SourceMap { return "no-location".to_string(); } - let lo = self.lookup_char_pos_adj(sp.lo()); - let hi = self.lookup_char_pos_adj(sp.hi()); + let lo = self.lookup_char_pos(sp.lo()); + let hi = self.lookup_char_pos(sp.hi()); format!("{}:{}:{}: {}:{}", - lo.filename, + lo.file.name, lo.line, lo.col.to_usize() + 1, hi.line, @@ -508,7 +512,7 @@ impl SourceMap { start_col, end_col: hi.col }); - Ok(FileLines {file: lo.file, lines: lines}) + Ok(FileLines {file: lo.file, lines}) } /// Extracts the source surrounding the given `Span` using the `extract_source` function. The @@ -737,6 +741,11 @@ impl SourceMap { debug!("find_width_of_character_at_span: local_begin=`{:?}`, local_end=`{:?}`", local_begin, local_end); + if local_begin.sf.start_pos != local_end.sf.start_pos { + debug!("find_width_of_character_at_span: begin and end are in different files"); + return 1; + } + let start_index = local_begin.pos.to_usize(); let end_index = local_end.pos.to_usize(); debug!("find_width_of_character_at_span: start_index=`{:?}`, end_index=`{:?}`", @@ -812,7 +821,7 @@ impl SourceMap { let idx = self.lookup_source_file_idx(bpos); let sf = (*self.files.borrow().source_files)[idx].clone(); let offset = bpos - sf.start_pos; - SourceFileAndBytePos {sf: sf, pos: offset} + SourceFileAndBytePos {sf, pos: offset} } /// Converts an absolute BytePos to a CharPos relative to the source_file. @@ -1014,223 +1023,3 @@ impl FilePathMapping { (path, false) } } - -// _____________________________________________________________________________ -// Tests -// - -#[cfg(test)] -mod tests { - use super::*; - use rustc_data_structures::sync::Lrc; - - fn init_source_map() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - sm.new_source_file(PathBuf::from("blork.rs").into(), - "first line.\nsecond line".to_string()); - sm.new_source_file(PathBuf::from("empty.rs").into(), - String::new()); - sm.new_source_file(PathBuf::from("blork2.rs").into(), - "first line.\nsecond line".to_string()); - sm - } - - #[test] - fn t3() { - // Test lookup_byte_offset - let sm = init_source_map(); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(23)); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(0)); - - let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); - assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); - assert_eq!(srcfbp2.pos, BytePos(0)); - } - - #[test] - fn t4() { - // Test bytepos_to_file_charpos - let sm = init_source_map(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); - assert_eq!(cp1, CharPos(22)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); - assert_eq!(cp2, CharPos(0)); - } - - #[test] - fn t5() { - // Test zero-length source_files. - let sm = init_source_map(); - - let loc1 = sm.lookup_char_pos(BytePos(22)); - assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(loc1.line, 2); - assert_eq!(loc1.col, CharPos(10)); - - let loc2 = sm.lookup_char_pos(BytePos(25)); - assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); - assert_eq!(loc2.line, 1); - assert_eq!(loc2.col, CharPos(0)); - } - - fn init_source_map_mbc() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - // € is a three byte utf8 char. - sm.new_source_file(PathBuf::from("blork.rs").into(), - "fir€st €€€€ line.\nsecond line".to_string()); - sm.new_source_file(PathBuf::from("blork2.rs").into(), - "first line€€.\n€ second line".to_string()); - sm - } - - #[test] - fn t6() { - // Test bytepos_to_file_charpos in the presence of multi-byte chars - let sm = init_source_map_mbc(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); - assert_eq!(cp1, CharPos(3)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); - assert_eq!(cp2, CharPos(4)); - - let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); - assert_eq!(cp3, CharPos(12)); - - let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); - assert_eq!(cp4, CharPos(15)); - } - - #[test] - fn t7() { - // Test span_to_lines for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let file_lines = sm.span_to_lines(span).unwrap(); - - assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(file_lines.lines.len(), 1); - assert_eq!(file_lines.lines[0].line_index, 1); - } - - /// Given a string like " ~~~~~~~~~~~~ ", produces a span - /// converting that range. The idea is that the string has the same - /// length as the input, and we uncover the byte positions. Note - /// that this can span lines and so on. - fn span_from_selection(input: &str, selection: &str) -> Span { - assert_eq!(input.len(), selection.len()); - let left_index = selection.find('~').unwrap() as u32; - let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index); - Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION) - } - - /// Tests span_to_snippet and span_to_lines for a span converting 3 - /// lines in the middle of a file. - #[test] - fn span_to_snippet_and_lines_spanning_multiple_lines() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; - let selection = " \n ~~\n~~~\n~~~~~ \n \n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); - let span = span_from_selection(inputtext, selection); - - // check that we are extracting the text we thought we were extracting - assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); - - // check that span_to_lines gives us the complete result with the lines/cols we expected - let lines = sm.span_to_lines(span).unwrap(); - let expected = vec![ - LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, - LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, - LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) } - ]; - assert_eq!(lines.lines, expected); - } - - #[test] - fn t8() { - // Test span_to_snippet for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let snippet = sm.span_to_snippet(span); - - assert_eq!(snippet, Ok("second line".to_string())); - } - - #[test] - fn t9() { - // Test span_to_str for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let sstr = sm.span_to_string(span); - - assert_eq!(sstr, "blork.rs:2:1: 2:12"); - } - - /// Tests failing to merge two spans on different lines - #[test] - fn span_merging_fail() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "bbbb BB\ncc CCC\n"; - let selection1 = " ~~\n \n"; - let selection2 = " \n ~~~\n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); - let span1 = span_from_selection(inputtext, selection1); - let span2 = span_from_selection(inputtext, selection2); - - assert!(sm.merge_spans(span1, span2).is_none()); - } - - /// Returns the span corresponding to the `n`th occurrence of - /// `substring` in `source_text`. - trait SourceMapExtension { - fn span_substr(&self, - file: &Lrc<SourceFile>, - source_text: &str, - substring: &str, - n: usize) - -> Span; - } - - impl SourceMapExtension for SourceMap { - fn span_substr(&self, - file: &Lrc<SourceFile>, - source_text: &str, - substring: &str, - n: usize) - -> Span - { - println!("span_substr(file={:?}/{:?}, substring={:?}, n={})", - file.name, file.start_pos, substring, n); - let mut i = 0; - let mut hi = 0; - loop { - let offset = source_text[hi..].find(substring).unwrap_or_else(|| { - panic!("source_text `{}` does not have {} occurrences of `{}`, only {}", - source_text, n, substring, i); - }); - let lo = hi + offset; - hi = lo + substring.len(); - if i == n { - let span = Span::new( - BytePos(lo as u32 + file.start_pos.0), - BytePos(hi as u32 + file.start_pos.0), - NO_EXPANSION, - ); - assert_eq!(&self.span_to_snippet(span).unwrap()[..], - substring); - return span; - } - i += 1; - } - } - } -} diff --git a/src/libsyntax/source_map/tests.rs b/src/libsyntax/source_map/tests.rs new file mode 100644 index 00000000000..427e86b56e1 --- /dev/null +++ b/src/libsyntax/source_map/tests.rs @@ -0,0 +1,213 @@ +use super::*; + +use rustc_data_structures::sync::Lrc; + +fn init_source_map() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + sm.new_source_file(PathBuf::from("blork.rs").into(), + "first line.\nsecond line".to_string()); + sm.new_source_file(PathBuf::from("empty.rs").into(), + String::new()); + sm.new_source_file(PathBuf::from("blork2.rs").into(), + "first line.\nsecond line".to_string()); + sm +} + +#[test] +fn t3() { + // Test lookup_byte_offset + let sm = init_source_map(); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(23)); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(0)); + + let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); + assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); + assert_eq!(srcfbp2.pos, BytePos(0)); +} + +#[test] +fn t4() { + // Test bytepos_to_file_charpos + let sm = init_source_map(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); + assert_eq!(cp1, CharPos(22)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); + assert_eq!(cp2, CharPos(0)); +} + +#[test] +fn t5() { + // Test zero-length source_files. + let sm = init_source_map(); + + let loc1 = sm.lookup_char_pos(BytePos(22)); + assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(loc1.line, 2); + assert_eq!(loc1.col, CharPos(10)); + + let loc2 = sm.lookup_char_pos(BytePos(25)); + assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); + assert_eq!(loc2.line, 1); + assert_eq!(loc2.col, CharPos(0)); +} + +fn init_source_map_mbc() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + // € is a three byte utf8 char. + sm.new_source_file(PathBuf::from("blork.rs").into(), + "fir€st €€€€ line.\nsecond line".to_string()); + sm.new_source_file(PathBuf::from("blork2.rs").into(), + "first line€€.\n€ second line".to_string()); + sm +} + +#[test] +fn t6() { + // Test bytepos_to_file_charpos in the presence of multi-byte chars + let sm = init_source_map_mbc(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); + assert_eq!(cp1, CharPos(3)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); + assert_eq!(cp2, CharPos(4)); + + let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); + assert_eq!(cp3, CharPos(12)); + + let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); + assert_eq!(cp4, CharPos(15)); +} + +#[test] +fn t7() { + // Test span_to_lines for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let file_lines = sm.span_to_lines(span).unwrap(); + + assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(file_lines.lines.len(), 1); + assert_eq!(file_lines.lines[0].line_index, 1); +} + +/// Given a string like " ~~~~~~~~~~~~ ", produces a span +/// converting that range. The idea is that the string has the same +/// length as the input, and we uncover the byte positions. Note +/// that this can span lines and so on. +fn span_from_selection(input: &str, selection: &str) -> Span { + assert_eq!(input.len(), selection.len()); + let left_index = selection.find('~').unwrap() as u32; + let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index); + Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION) +} + +/// Tests span_to_snippet and span_to_lines for a span converting 3 +/// lines in the middle of a file. +#[test] +fn span_to_snippet_and_lines_spanning_multiple_lines() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; + let selection = " \n ~~\n~~~\n~~~~~ \n \n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); + let span = span_from_selection(inputtext, selection); + + // check that we are extracting the text we thought we were extracting + assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); + + // check that span_to_lines gives us the complete result with the lines/cols we expected + let lines = sm.span_to_lines(span).unwrap(); + let expected = vec![ + LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, + LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, + LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) } + ]; + assert_eq!(lines.lines, expected); +} + +#[test] +fn t8() { + // Test span_to_snippet for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let snippet = sm.span_to_snippet(span); + + assert_eq!(snippet, Ok("second line".to_string())); +} + +#[test] +fn t9() { + // Test span_to_str for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let sstr = sm.span_to_string(span); + + assert_eq!(sstr, "blork.rs:2:1: 2:12"); +} + +/// Tests failing to merge two spans on different lines +#[test] +fn span_merging_fail() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "bbbb BB\ncc CCC\n"; + let selection1 = " ~~\n \n"; + let selection2 = " \n ~~~\n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); + let span1 = span_from_selection(inputtext, selection1); + let span2 = span_from_selection(inputtext, selection2); + + assert!(sm.merge_spans(span1, span2).is_none()); +} + +/// Returns the span corresponding to the `n`th occurrence of +/// `substring` in `source_text`. +trait SourceMapExtension { + fn span_substr(&self, + file: &Lrc<SourceFile>, + source_text: &str, + substring: &str, + n: usize) + -> Span; +} + +impl SourceMapExtension for SourceMap { + fn span_substr(&self, + file: &Lrc<SourceFile>, + source_text: &str, + substring: &str, + n: usize) + -> Span + { + println!("span_substr(file={:?}/{:?}, substring={:?}, n={})", + file.name, file.start_pos, substring, n); + let mut i = 0; + let mut hi = 0; + loop { + let offset = source_text[hi..].find(substring).unwrap_or_else(|| { + panic!("source_text `{}` does not have {} occurrences of `{}`, only {}", + source_text, n, substring, i); + }); + let lo = hi + offset; + hi = lo + substring.len(); + if i == n { + let span = Span::new( + BytePos(lo as u32 + file.start_pos.0), + BytePos(hi as u32 + file.start_pos.0), + NO_EXPANSION, + ); + assert_eq!(&self.span_to_snippet(span).unwrap()[..], + substring); + return span; + } + i += 1; + } + } +} diff --git a/src/libsyntax/std_inject.rs b/src/libsyntax/std_inject.rs deleted file mode 100644 index b9758bd655c..00000000000 --- a/src/libsyntax/std_inject.rs +++ /dev/null @@ -1,125 +0,0 @@ -use crate::ast; -use crate::attr; -use crate::edition::Edition; -use crate::ext::hygiene::{Mark, SyntaxContext}; -use crate::symbol::{Symbol, keywords}; -use crate::source_map::{ExpnInfo, MacroAttribute, dummy_spanned, hygiene, respan}; -use crate::ptr::P; -use crate::tokenstream::TokenStream; - -use std::cell::Cell; -use std::iter; -use syntax_pos::{DUMMY_SP, Span}; - -/// Craft a span that will be ignored by the stability lint's -/// call to source_map's `is_internal` check. -/// The expanded code uses the unstable `#[prelude_import]` attribute. -fn ignored_span(sp: Span) -> Span { - let mark = Mark::fresh(Mark::root()); - mark.set_expn_info(ExpnInfo { - call_site: DUMMY_SP, - def_site: None, - format: MacroAttribute(Symbol::intern("std_inject")), - allow_internal_unstable: Some(vec![ - Symbol::intern("prelude_import"), - ].into()), - allow_internal_unsafe: false, - local_inner_macros: false, - edition: hygiene::default_edition(), - }); - sp.with_ctxt(SyntaxContext::empty().apply_mark(mark)) -} - -pub fn injected_crate_name() -> Option<&'static str> { - INJECTED_CRATE_NAME.with(|name| name.get()) -} - -thread_local! { - static INJECTED_CRATE_NAME: Cell<Option<&'static str>> = Cell::new(None); -} - -pub fn maybe_inject_crates_ref( - mut krate: ast::Crate, - alt_std_name: Option<&str>, - edition: Edition, -) -> ast::Crate { - let rust_2018 = edition >= Edition::Edition2018; - - // the first name in this list is the crate name of the crate with the prelude - let names: &[&str] = if attr::contains_name(&krate.attrs, "no_core") { - return krate; - } else if attr::contains_name(&krate.attrs, "no_std") { - if attr::contains_name(&krate.attrs, "compiler_builtins") { - &["core"] - } else { - &["core", "compiler_builtins"] - } - } else { - &["std"] - }; - - // .rev() to preserve ordering above in combination with insert(0, ...) - let alt_std_name = alt_std_name.map(Symbol::intern); - for orig_name in names.iter().rev() { - let orig_name = Symbol::intern(orig_name); - let mut rename = orig_name; - // HACK(eddyb) gensym the injected crates on the Rust 2018 edition, - // so they don't accidentally interfere with the new import paths. - if rust_2018 { - rename = orig_name.gensymed(); - } - let orig_name = if rename != orig_name { - Some(orig_name) - } else { - None - }; - krate.module.items.insert(0, P(ast::Item { - attrs: vec![attr::mk_attr_outer(DUMMY_SP, - attr::mk_attr_id(), - attr::mk_word_item(ast::Ident::from_str("macro_use")))], - vis: dummy_spanned(ast::VisibilityKind::Inherited), - node: ast::ItemKind::ExternCrate(alt_std_name.or(orig_name)), - ident: ast::Ident::with_empty_ctxt(rename), - id: ast::DUMMY_NODE_ID, - span: DUMMY_SP, - tokens: None, - })); - } - - // the crates have been injected, the assumption is that the first one is the one with - // the prelude. - let name = names[0]; - - INJECTED_CRATE_NAME.with(|opt_name| opt_name.set(Some(name))); - - let span = ignored_span(DUMMY_SP); - krate.module.items.insert(0, P(ast::Item { - attrs: vec![ast::Attribute { - style: ast::AttrStyle::Outer, - path: ast::Path::from_ident(ast::Ident::new(Symbol::intern("prelude_import"), span)), - tokens: TokenStream::empty(), - id: attr::mk_attr_id(), - is_sugared_doc: false, - span, - }], - vis: respan(span.shrink_to_lo(), ast::VisibilityKind::Inherited), - node: ast::ItemKind::Use(P(ast::UseTree { - prefix: ast::Path { - segments: iter::once(keywords::PathRoot.ident()) - .chain( - [name, "prelude", "v1"].iter().cloned() - .map(ast::Ident::from_str) - ).map(ast::PathSegment::from_ident).collect(), - span, - }, - kind: ast::UseTreeKind::Glob, - span, - })), - id: ast::DUMMY_NODE_ID, - ident: keywords::Invalid.ident(), - span, - tokens: None, - })); - - krate -} diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs deleted file mode 100644 index 56290fa771b..00000000000 --- a/src/libsyntax/test.rs +++ /dev/null @@ -1,443 +0,0 @@ -// Code that generates a test runner to run all the tests in a crate - -#![allow(dead_code)] -#![allow(unused_imports)] - -use HasTestSignature::*; - -use std::iter; -use std::slice; -use std::mem; -use std::vec; - -use log::debug; -use smallvec::{smallvec, SmallVec}; -use syntax_pos::{DUMMY_SP, NO_EXPANSION, Span, SourceFile, BytePos}; - -use crate::attr::{self, HasAttrs}; -use crate::source_map::{self, SourceMap, ExpnInfo, MacroAttribute, dummy_spanned, respan}; -use crate::config; -use crate::entry::{self, EntryPointType}; -use crate::ext::base::{ExtCtxt, Resolver}; -use crate::ext::build::AstBuilder; -use crate::ext::expand::ExpansionConfig; -use crate::ext::hygiene::{self, Mark, SyntaxContext}; -use crate::mut_visit::{*, ExpectOne}; -use crate::feature_gate::Features; -use crate::util::map_in_place::MapInPlace; -use crate::parse::{token, ParseSess}; -use crate::print::pprust; -use crate::ast::{self, Ident}; -use crate::ptr::P; -use crate::symbol::{self, Symbol, keywords}; -use crate::ThinVec; - -struct Test { - span: Span, - path: Vec<Ident>, -} - -struct TestCtxt<'a> { - span_diagnostic: &'a errors::Handler, - path: Vec<Ident>, - ext_cx: ExtCtxt<'a>, - test_cases: Vec<Test>, - reexport_test_harness_main: Option<Symbol>, - is_libtest: bool, - ctxt: SyntaxContext, - features: &'a Features, - test_runner: Option<ast::Path>, - - // top-level re-export submodule, filled out after folding is finished - toplevel_reexport: Option<Ident>, -} - -// Traverse the crate, collecting all the test functions, eliding any -// existing main functions, and synthesizing a main test harness -pub fn modify_for_testing(sess: &ParseSess, - resolver: &mut dyn Resolver, - should_test: bool, - krate: &mut ast::Crate, - span_diagnostic: &errors::Handler, - features: &Features) { - // Check for #[reexport_test_harness_main = "some_name"] which - // creates a `use __test::main as some_name;`. This needs to be - // unconditional, so that the attribute is still marked as used in - // non-test builds. - let reexport_test_harness_main = - attr::first_attr_value_str_by_name(&krate.attrs, - "reexport_test_harness_main"); - - // Do this here so that the test_runner crate attribute gets marked as used - // even in non-test builds - let test_runner = get_test_runner(span_diagnostic, &krate); - - if should_test { - generate_test_harness(sess, resolver, reexport_test_harness_main, - krate, span_diagnostic, features, test_runner) - } -} - -struct TestHarnessGenerator<'a> { - cx: TestCtxt<'a>, - tests: Vec<Ident>, - - // submodule name, gensym'd identifier for re-exports - tested_submods: Vec<(Ident, Ident)>, -} - -impl<'a> MutVisitor for TestHarnessGenerator<'a> { - fn visit_crate(&mut self, c: &mut ast::Crate) { - noop_visit_crate(c, self); - - // Create a main function to run our tests - let test_main = { - let unresolved = mk_main(&mut self.cx); - self.cx.ext_cx.monotonic_expander().flat_map_item(unresolved).pop().unwrap() - }; - - c.module.items.push(test_main); - } - - fn flat_map_item(&mut self, i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { - let ident = i.ident; - if ident.name != keywords::Invalid.name() { - self.cx.path.push(ident); - } - debug!("current path: {}", path_name_i(&self.cx.path)); - - let mut item = i.into_inner(); - if is_test_case(&item) { - debug!("this is a test item"); - - let test = Test { - span: item.span, - path: self.cx.path.clone(), - }; - self.cx.test_cases.push(test); - self.tests.push(item.ident); - } - - // We don't want to recurse into anything other than mods, since - // mods or tests inside of functions will break things - if let ast::ItemKind::Mod(mut module) = item.node { - let tests = mem::replace(&mut self.tests, Vec::new()); - let tested_submods = mem::replace(&mut self.tested_submods, Vec::new()); - noop_visit_mod(&mut module, self); - let tests = mem::replace(&mut self.tests, tests); - let tested_submods = mem::replace(&mut self.tested_submods, tested_submods); - - if !tests.is_empty() || !tested_submods.is_empty() { - let (it, sym) = mk_reexport_mod(&mut self.cx, item.id, tests, tested_submods); - module.items.push(it); - - if !self.cx.path.is_empty() { - self.tested_submods.push((self.cx.path[self.cx.path.len()-1], sym)); - } else { - debug!("pushing nothing, sym: {:?}", sym); - self.cx.toplevel_reexport = Some(sym); - } - } - item.node = ast::ItemKind::Mod(module); - } - if ident.name != keywords::Invalid.name() { - self.cx.path.pop(); - } - smallvec![P(item)] - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} - -/// A folder used to remove any entry points (like fn main) because the harness -/// generator will provide its own -struct EntryPointCleaner { - // Current depth in the ast - depth: usize, -} - -impl MutVisitor for EntryPointCleaner { - fn flat_map_item(&mut self, i: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> { - self.depth += 1; - let item = noop_flat_map_item(i, self).expect_one("noop did something"); - self.depth -= 1; - - // Remove any #[main] or #[start] from the AST so it doesn't - // clash with the one we're going to add, but mark it as - // #[allow(dead_code)] to avoid printing warnings. - let item = match entry::entry_point_type(&item, self.depth) { - EntryPointType::MainNamed | - EntryPointType::MainAttr | - EntryPointType::Start => - item.map(|ast::Item {id, ident, attrs, node, vis, span, tokens}| { - let allow_ident = Ident::from_str("allow"); - let dc_nested = attr::mk_nested_word_item(Ident::from_str("dead_code")); - let allow_dead_code_item = attr::mk_list_item(DUMMY_SP, allow_ident, - vec![dc_nested]); - let allow_dead_code = attr::mk_attr_outer(DUMMY_SP, - attr::mk_attr_id(), - allow_dead_code_item); - - ast::Item { - id, - ident, - attrs: attrs.into_iter() - .filter(|attr| { - !attr.check_name("main") && !attr.check_name("start") - }) - .chain(iter::once(allow_dead_code)) - .collect(), - node, - vis, - span, - tokens, - } - }), - EntryPointType::None | - EntryPointType::OtherMain => item, - }; - - smallvec![item] - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Do nothing. - } -} - -/// Creates an item (specifically a module) that "pub use"s the tests passed in. -/// Each tested submodule will contain a similar reexport module that we will export -/// under the name of the original module. That is, `submod::__test_reexports` is -/// reexported like so `pub use submod::__test_reexports as submod`. -fn mk_reexport_mod(cx: &mut TestCtxt<'_>, - parent: ast::NodeId, - tests: Vec<Ident>, - tested_submods: Vec<(Ident, Ident)>) - -> (P<ast::Item>, Ident) { - let super_ = Ident::from_str("super"); - - let items = tests.into_iter().map(|r| { - cx.ext_cx.item_use_simple(DUMMY_SP, dummy_spanned(ast::VisibilityKind::Public), - cx.ext_cx.path(DUMMY_SP, vec![super_, r])) - }).chain(tested_submods.into_iter().map(|(r, sym)| { - let path = cx.ext_cx.path(DUMMY_SP, vec![super_, r, sym]); - cx.ext_cx.item_use_simple_(DUMMY_SP, dummy_spanned(ast::VisibilityKind::Public), - Some(r), path) - })).collect(); - - let reexport_mod = ast::Mod { - inline: true, - inner: DUMMY_SP, - items, - }; - - let sym = Ident::with_empty_ctxt(Symbol::gensym("__test_reexports")); - let parent = if parent == ast::DUMMY_NODE_ID { ast::CRATE_NODE_ID } else { parent }; - cx.ext_cx.current_expansion.mark = cx.ext_cx.resolver.get_module_scope(parent); - let it = cx.ext_cx.monotonic_expander().flat_map_item(P(ast::Item { - ident: sym, - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - node: ast::ItemKind::Mod(reexport_mod), - vis: dummy_spanned(ast::VisibilityKind::Public), - span: DUMMY_SP, - tokens: None, - })).pop().unwrap(); - - (it, sym) -} - -/// Crawl over the crate, inserting test reexports and the test main function -fn generate_test_harness(sess: &ParseSess, - resolver: &mut dyn Resolver, - reexport_test_harness_main: Option<Symbol>, - krate: &mut ast::Crate, - sd: &errors::Handler, - features: &Features, - test_runner: Option<ast::Path>) { - // Remove the entry points - let mut cleaner = EntryPointCleaner { depth: 0 }; - cleaner.visit_crate(krate); - - let mark = Mark::fresh(Mark::root()); - - let mut econfig = ExpansionConfig::default("test".to_string()); - econfig.features = Some(features); - - let cx = TestCtxt { - span_diagnostic: sd, - ext_cx: ExtCtxt::new(sess, econfig, resolver), - path: Vec::new(), - test_cases: Vec::new(), - reexport_test_harness_main, - // N.B., doesn't consider the value of `--crate-name` passed on the command line. - is_libtest: attr::find_crate_name(&krate.attrs).map(|s| s == "test").unwrap_or(false), - toplevel_reexport: None, - ctxt: SyntaxContext::empty().apply_mark(mark), - features, - test_runner - }; - - mark.set_expn_info(ExpnInfo { - call_site: DUMMY_SP, - def_site: None, - format: MacroAttribute(Symbol::intern("test_case")), - allow_internal_unstable: Some(vec![ - Symbol::intern("main"), - Symbol::intern("test"), - Symbol::intern("rustc_attrs"), - ].into()), - allow_internal_unsafe: false, - local_inner_macros: false, - edition: hygiene::default_edition(), - }); - - TestHarnessGenerator { - cx, - tests: Vec::new(), - tested_submods: Vec::new(), - }.visit_crate(krate); -} - -/// Craft a span that will be ignored by the stability lint's -/// call to source_map's `is_internal` check. -/// The expanded code calls some unstable functions in the test crate. -fn ignored_span(cx: &TestCtxt<'_>, sp: Span) -> Span { - sp.with_ctxt(cx.ctxt) -} - -enum HasTestSignature { - Yes, - No(BadTestSignature), -} - -#[derive(PartialEq)] -enum BadTestSignature { - NotEvenAFunction, - WrongTypeSignature, - NoArgumentsAllowed, - ShouldPanicOnlyWithNoArgs, -} - -/// Creates a function item for use as the main function of a test build. -/// This function will call the `test_runner` as specified by the crate attribute -fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> { - // Writing this out by hand with 'ignored_span': - // pub fn main() { - // #![main] - // test::test_main_static(::std::os::args().as_slice(), &[..tests]); - // } - let sp = ignored_span(cx, DUMMY_SP); - let ecx = &cx.ext_cx; - let test_id = ecx.ident_of("test").gensym(); - - // test::test_main_static(...) - let mut test_runner = cx.test_runner.clone().unwrap_or( - ecx.path(sp, vec![ - test_id, ecx.ident_of("test_main_static") - ])); - - test_runner.span = sp; - - let test_main_path_expr = ecx.expr_path(test_runner); - let call_test_main = ecx.expr_call(sp, test_main_path_expr, - vec![mk_tests_slice(cx)]); - let call_test_main = ecx.stmt_expr(call_test_main); - - // #![main] - let main_meta = ecx.meta_word(sp, Symbol::intern("main")); - let main_attr = ecx.attribute(sp, main_meta); - - // extern crate test as test_gensym - let test_extern_stmt = ecx.stmt_item(sp, ecx.item(sp, - test_id, - vec![], - ast::ItemKind::ExternCrate(Some(Symbol::intern("test"))) - )); - - // pub fn main() { ... } - let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(vec![])); - - // If no test runner is provided we need to import the test crate - let main_body = if cx.test_runner.is_none() { - ecx.block(sp, vec![test_extern_stmt, call_test_main]) - } else { - ecx.block(sp, vec![call_test_main]) - }; - - let main = ast::ItemKind::Fn(ecx.fn_decl(vec![], ast::FunctionRetTy::Ty(main_ret_ty)), - ast::FnHeader::default(), - ast::Generics::default(), - main_body); - - // Honor the reexport_test_harness_main attribute - let main_id = Ident::new( - cx.reexport_test_harness_main.unwrap_or(Symbol::gensym("main")), - sp); - - P(ast::Item { - ident: main_id, - attrs: vec![main_attr], - id: ast::DUMMY_NODE_ID, - node: main, - vis: dummy_spanned(ast::VisibilityKind::Public), - span: sp, - tokens: None, - }) - -} - -fn path_name_i(idents: &[Ident]) -> String { - let mut path_name = "".to_string(); - let mut idents_iter = idents.iter().peekable(); - while let Some(ident) = idents_iter.next() { - path_name.push_str(&ident.as_str()); - if idents_iter.peek().is_some() { - path_name.push_str("::") - } - } - path_name -} - -/// Creates a slice containing every test like so: -/// &[path::to::test1, path::to::test2] -fn mk_tests_slice(cx: &TestCtxt<'_>) -> P<ast::Expr> { - debug!("building test vector from {} tests", cx.test_cases.len()); - let ref ecx = cx.ext_cx; - - ecx.expr_vec_slice(DUMMY_SP, - cx.test_cases.iter().map(|test| { - ecx.expr_addr_of(test.span, - ecx.expr_path(ecx.path(test.span, visible_path(cx, &test.path)))) - }).collect()) -} - -/// Creates a path from the top-level __test module to the test via __test_reexports -fn visible_path(cx: &TestCtxt<'_>, path: &[Ident]) -> Vec<Ident>{ - let mut visible_path = vec![]; - match cx.toplevel_reexport { - Some(id) => visible_path.push(id), - None => { - cx.span_diagnostic.bug("expected to find top-level re-export name, but found None"); - } - } - visible_path.extend_from_slice(path); - visible_path -} - -fn is_test_case(i: &ast::Item) -> bool { - attr::contains_name(&i.attrs, "rustc_test_marker") -} - -fn get_test_runner(sd: &errors::Handler, krate: &ast::Crate) -> Option<ast::Path> { - let test_attr = attr::find_by_name(&krate.attrs, "test_runner")?; - test_attr.meta_item_list().map(|meta_list| { - if meta_list.len() != 1 { - sd.span_fatal(test_attr.span(), - "#![test_runner(..)] accepts exactly 1 argument").raise() - } - meta_list[0].word().as_ref().unwrap().ident.clone() - }) -} diff --git a/src/libsyntax/test_snippet.rs b/src/libsyntax/tests.rs index 2b3d18835d5..cff034fdeb1 100644 --- a/src/libsyntax/test_snippet.rs +++ b/src/libsyntax/tests.rs @@ -1,16 +1,106 @@ +use crate::{ast, panictry}; +use crate::parse::{ParseSess, PResult, source_file_to_stream}; +use crate::parse::new_parser_from_source_str; +use crate::parse::parser::Parser; use crate::source_map::{SourceMap, FilePathMapping}; -use crate::with_globals; +use crate::tokenstream::TokenStream; +use crate::with_default_globals; -use errors::Handler; use errors::emitter::EmitterWriter; +use errors::Handler; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan}; use std::io; use std::io::prelude::*; -use rustc_data_structures::sync::Lrc; +use std::iter::Peekable; +use std::path::{Path, PathBuf}; use std::str; use std::sync::{Arc, Mutex}; -use std::path::Path; -use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan}; + +/// Map string to parser (via tts) +fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { + new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) +} + +crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where + F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, +{ + let mut p = string_to_parser(&ps, s); + let x = panictry!(f(&mut p)); + p.sess.span_diagnostic.abort_if_errors(); + x +} + +/// Map a string to tts, using a made-up filename: +crate fn string_to_stream(source_str: String) -> TokenStream { + let ps = ParseSess::new(FilePathMapping::empty()); + source_file_to_stream( + &ps, + ps.source_map().new_source_file(PathBuf::from("bogofile").into(), + source_str, + ), None).0 +} + +/// Parse a string, return a crate. +crate fn string_to_crate(source_str : String) -> ast::Crate { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_crate_mod() + }) +} + +/// Does the given string match the pattern? whitespace in the first string +/// may be deleted or replaced with other whitespace to match the pattern. +/// This function is relatively Unicode-ignorant; fortunately, the careful design +/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). +crate fn matches_codepattern(a : &str, b : &str) -> bool { + let mut a_iter = a.chars().peekable(); + let mut b_iter = b.chars().peekable(); + + loop { + let (a, b) = match (a_iter.peek(), b_iter.peek()) { + (None, None) => return true, + (None, _) => return false, + (Some(&a), None) => { + if is_pattern_whitespace(a) { + break // trailing whitespace check is out of loop for borrowck + } else { + return false + } + } + (Some(&a), Some(&b)) => (a, b) + }; + + if is_pattern_whitespace(a) && is_pattern_whitespace(b) { + // skip whitespace for a and b + scan_for_non_ws_or_end(&mut a_iter); + scan_for_non_ws_or_end(&mut b_iter); + } else if is_pattern_whitespace(a) { + // skip whitespace for a + scan_for_non_ws_or_end(&mut a_iter); + } else if a == b { + a_iter.next(); + b_iter.next(); + } else { + return false + } + } + + // check if a has *only* trailing whitespace + a_iter.all(is_pattern_whitespace) +} + +/// Advances the given peekable `Iterator` until it reaches a non-whitespace character +fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) { + while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) { + iter.next(); + } +} + +fn is_pattern_whitespace(c: char) -> bool { + rustc_lexer::character_properties::is_whitespace(c) +} /// Identify a position in the text by the Nth occurrence of a string. struct Position { @@ -39,7 +129,7 @@ impl<T: Write> Write for Shared<T> { } fn test_harness(file_text: &str, span_labels: Vec<SpanLabel>, expected_output: &str) { - with_globals(|| { + with_default_globals(|| { let output = Arc::new(Mutex::new(Vec::new())); let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); @@ -57,6 +147,7 @@ fn test_harness(file_text: &str, span_labels: Vec<SpanLabel>, expected_output: & let emitter = EmitterWriter::new(Box::new(Shared { data: output.clone() }), Some(source_map.clone()), false, + false, false); let handler = Handler::with_emitter(true, None, Box::new(emitter)); handler.span_err(msp, "foo"); @@ -375,6 +466,66 @@ error: foo } #[test] +fn triple_exact_overlap() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`Z` label", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | / X0 Y0 Z0 +4 | | X1 Y1 Z1 +5 | | X2 Y2 Z2 + | | ^ + | | | + | | `X` is a good letter + | |____`Y` is a good letter too + | `Z` label + +"#); +} + +#[test] fn minimum_depth() { test_harness(r#" fn foo() { @@ -1101,4 +1252,3 @@ error: foo "#); } - diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 4ce308d015c..6ff8898fe21 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -16,18 +16,22 @@ use crate::ext::base; use crate::ext::tt::{macro_parser, quoted}; use crate::parse::Directory; -use crate::parse::token::{self, DelimToken, Token}; +use crate::parse::token::{self, DelimToken, Token, TokenKind}; use crate::print::pprust; -use syntax_pos::{BytePos, Mark, Span, DUMMY_SP}; +use syntax_pos::{BytePos, ExpnId, Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert; +use rustc_data_structures::static_assert_size; use rustc_data_structures::sync::Lrc; -use serialize::{Decoder, Decodable, Encoder, Encodable}; +use rustc_serialize::{Decoder, Decodable, Encoder, Encodable}; +use smallvec::{SmallVec, smallvec}; use std::borrow::Cow; use std::{fmt, iter, mem}; +#[cfg(test)] +mod tests; + /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -43,11 +47,21 @@ use std::{fmt, iter, mem}; #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum TokenTree { /// A single token - Token(Span, token::Token), + Token(Token), /// A delimited sequence of token trees Delimited(DelimSpan, DelimToken, TokenStream), } +// Ensure all fields of `TokenTree` is `Send` and `Sync`. +#[cfg(parallel_compiler)] +fn _dummy() +where + Token: Send + Sync, + DelimSpan: Send + Sync, + DelimToken: Send + Sync, + TokenStream: Send + Sync, +{} + impl TokenTree { /// Use this token tree as a matcher to parse given tts. pub fn parse(cx: &base::ExtCtxt<'_>, mtch: &[quoted::TokenTree], tts: TokenStream) @@ -63,12 +77,11 @@ impl TokenTree { /// Checks if this TokenTree is equal to the other, regardless of span information. pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { - (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, - (&TokenTree::Delimited(_, delim, ref tts), - &TokenTree::Delimited(_, delim2, ref tts2)) => { + (TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind, + (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { delim == delim2 && tts.eq_unspanned(&tts2) } - (_, _) => false, + _ => false, } } @@ -79,38 +92,29 @@ impl TokenTree { // different method. pub fn probably_equal_for_proc_macro(&self, other: &TokenTree) -> bool { match (self, other) { - (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => { - tk.probably_equal_for_proc_macro(tk2) + (TokenTree::Token(token), TokenTree::Token(token2)) => { + token.probably_equal_for_proc_macro(token2) } - (&TokenTree::Delimited(_, delim, ref tts), - &TokenTree::Delimited(_, delim2, ref tts2)) => { + (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { delim == delim2 && tts.probably_equal_for_proc_macro(&tts2) } - (_, _) => false, + _ => false, } } /// Retrieves the TokenTree's span. pub fn span(&self) -> Span { - match *self { - TokenTree::Token(sp, _) => sp, + match self { + TokenTree::Token(token) => token.span, TokenTree::Delimited(sp, ..) => sp.entire(), } } /// Modify the `TokenTree`'s span in-place. pub fn set_span(&mut self, span: Span) { - match *self { - TokenTree::Token(ref mut sp, _) => *sp = span, - TokenTree::Delimited(ref mut sp, ..) => *sp = DelimSpan::from_single(span), - } - } - - /// Indicates if the stream is a token that is equal to the provided token. - pub fn eq_token(&self, t: Token) -> bool { - match *self { - TokenTree::Token(_, ref tk) => *tk == t, - _ => false, + match self { + TokenTree::Token(token) => token.span = span, + TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), } } @@ -118,6 +122,10 @@ impl TokenTree { TokenStream::new(vec![(self, Joint)]) } + pub fn token(kind: TokenKind, span: Span) -> TokenTree { + TokenTree::Token(Token::new(kind, span)) + } + /// Returns the opening delimiter as a token tree. pub fn open_tt(span: Span, delim: DelimToken) -> TokenTree { let open_span = if span.is_dummy() { @@ -125,7 +133,7 @@ impl TokenTree { } else { span.with_hi(span.lo() + BytePos(delim.len() as u32)) }; - TokenTree::Token(open_span, token::OpenDelim(delim)) + TokenTree::token(token::OpenDelim(delim), open_span) } /// Returns the closing delimiter as a token tree. @@ -135,7 +143,7 @@ impl TokenTree { } else { span.with_lo(span.hi() - BytePos(delim.len() as u32)) }; - TokenTree::Token(close_span, token::CloseDelim(delim)) + TokenTree::token(token::CloseDelim(delim), close_span) } } @@ -144,7 +152,7 @@ impl TokenTree { /// A `TokenStream` is an abstract sequence of tokens, organized into `TokenTree`s. /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s /// instead of a representation of the abstract syntax tree. -/// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat. +/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat. /// /// The use of `Option` is an optimization that avoids the need for an /// allocation when the stream is empty. However, it is not guaranteed that an @@ -157,7 +165,7 @@ pub type TreeAndJoint = (TokenTree, IsJoint); // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::<TokenStream>() == 8); +static_assert_size!(TokenStream, 8); #[derive(Clone, Copy, Debug, PartialEq)] pub enum IsJoint { @@ -178,14 +186,18 @@ impl TokenStream { while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { - ((TokenTree::Token(_, token::Token::Comma), NonJoint), _) | - (_, (TokenTree::Token(_, token::Token::Comma), NonJoint)) => continue, - ((TokenTree::Token(sp, _), NonJoint), _) => *sp, + (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, + ((TokenTree::Token(token_left), NonJoint), + (TokenTree::Token(token_right), _)) + if ((token_left.is_ident() && !token_left.is_reserved_ident()) + || token_left.is_lit()) && + ((token_right.is_ident() && !token_right.is_reserved_ident()) + || token_right.is_lit()) => token_left.span, ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = (TokenTree::Token(sp, token::Comma), NonJoint); + let comma = (TokenTree::token(token::Comma, sp), NonJoint); suggestion = Some((pos, comma, sp)); } } @@ -214,15 +226,9 @@ impl From<TokenTree> for TreeAndJoint { } } -impl From<Token> for TokenStream { - fn from(token: Token) -> TokenStream { - TokenTree::Token(DUMMY_SP, token).into() - } -} - impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream { fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self { - TokenStream::from_streams(iter.into_iter().map(Into::into).collect::<Vec<_>>()) + TokenStream::from_streams(iter.into_iter().map(Into::into).collect::<SmallVec<_>>()) } } @@ -254,7 +260,7 @@ impl TokenStream { } } - pub(crate) fn from_streams(mut streams: Vec<TokenStream>) -> TokenStream { + pub(crate) fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { match streams.len() { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), @@ -322,22 +328,25 @@ impl TokenStream { // streams, making a comparison between a token stream generated from an // AST and a token stream which was parsed into an AST more reliable. fn semantic_tree(tree: &TokenTree) -> bool { - match tree { - // The pretty printer tends to add trailing commas to - // everything, and in particular, after struct fields. - | TokenTree::Token(_, Token::Comma) - // The pretty printer emits `NoDelim` as whitespace. - | TokenTree::Token(_, Token::OpenDelim(DelimToken::NoDelim)) - | TokenTree::Token(_, Token::CloseDelim(DelimToken::NoDelim)) - // The pretty printer collapses many semicolons into one. - | TokenTree::Token(_, Token::Semi) - // The pretty printer collapses whitespace arbitrarily and can - // introduce whitespace from `NoDelim`. - | TokenTree::Token(_, Token::Whitespace) - // The pretty printer can turn `$crate` into `::crate_name` - | TokenTree::Token(_, Token::ModSep) => false, - _ => true + if let TokenTree::Token(token) = tree { + if let + // The pretty printer tends to add trailing commas to + // everything, and in particular, after struct fields. + | token::Comma + // The pretty printer emits `NoDelim` as whitespace. + | token::OpenDelim(DelimToken::NoDelim) + | token::CloseDelim(DelimToken::NoDelim) + // The pretty printer collapses many semicolons into one. + | token::Semi + // The pretty printer collapses whitespace arbitrarily and can + // introduce whitespace from `NoDelim`. + | token::Whitespace + // The pretty printer can turn `$crate` into `::crate_name` + | token::ModSep = token.kind { + return false; + } } + true } let mut t1 = self.trees().filter(semantic_tree); @@ -391,24 +400,24 @@ impl TokenStream { } } +// 99.5%+ of the time we have 1 or 2 elements in this vector. #[derive(Clone)] -pub struct TokenStreamBuilder(Vec<TokenStream>); +pub struct TokenStreamBuilder(SmallVec<[TokenStream; 2]>); impl TokenStreamBuilder { pub fn new() -> TokenStreamBuilder { - TokenStreamBuilder(Vec::new()) + TokenStreamBuilder(SmallVec::new()) } pub fn push<T: Into<TokenStream>>(&mut self, stream: T) { let stream = stream.into(); let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint); - if let Some(TokenTree::Token(last_span, last_tok)) = last_tree_if_joint { - if let Some((TokenTree::Token(span, tok), is_joint)) = stream.first_tree_and_joint() { - if let Some(glued_tok) = last_tok.glue(tok) { + if let Some(TokenTree::Token(last_token)) = last_tree_if_joint { + if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() { + if let Some(glued_tok) = last_token.glue(token) { let last_stream = self.0.pop().unwrap(); self.push_all_but_last_tree(&last_stream); - let glued_span = last_span.to(span); - let glued_tt = TokenTree::Token(glued_span, glued_tok); + let glued_tt = TokenTree::Token(glued_tok); let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]); self.0.push(glued_tokenstream); self.push_all_but_first_tree(&stream); @@ -483,7 +492,7 @@ impl Cursor { } let index = self.index; let stream = mem::replace(&mut self.stream, TokenStream(None)); - *self = TokenStream::from_streams(vec![stream, new_stream]).into_trees(); + *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); self.index = index; } @@ -539,120 +548,10 @@ impl DelimSpan { self.open.with_hi(self.close.hi()) } - pub fn apply_mark(self, mark: Mark) -> Self { + pub fn apply_mark(self, expn_id: ExpnId) -> Self { DelimSpan { - open: self.open.apply_mark(mark), - close: self.close.apply_mark(mark), + open: self.open.apply_mark(expn_id), + close: self.close.apply_mark(expn_id), } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::syntax::ast::Ident; - use crate::with_globals; - use crate::parse::token::Token; - use crate::util::parser_testing::string_to_stream; - use syntax_pos::{Span, BytePos, NO_EXPANSION}; - - fn string_to_ts(string: &str) -> TokenStream { - string_to_stream(string.to_owned()) - } - - fn sp(a: u32, b: u32) -> Span { - Span::new(BytePos(a), BytePos(b), NO_EXPANSION) - } - - #[test] - fn test_concat() { - with_globals(|| { - let test_res = string_to_ts("foo::bar::baz"); - let test_fst = string_to_ts("foo::bar"); - let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(vec![test_fst, test_snd]); - assert_eq!(test_res.trees().count(), 5); - assert_eq!(eq_res.trees().count(), 5); - assert_eq!(test_res.eq_unspanned(&eq_res), true); - }) - } - - #[test] - fn test_to_from_bijection() { - with_globals(|| { - let test_start = string_to_ts("foo::bar(baz)"); - let test_end = test_start.trees().collect(); - assert_eq!(test_start, test_end) - }) - } - - #[test] - fn test_eq_0() { - with_globals(|| { - let test_res = string_to_ts("foo"); - let test_eqs = string_to_ts("foo"); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_eq_1() { - with_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("::bar::baz"); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_eq_3() { - with_globals(|| { - let test_res = string_to_ts(""); - let test_eqs = string_to_ts(""); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_diseq_0() { - with_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("bar::baz"); - assert_eq!(test_res == test_eqs, false) - }) - } - - #[test] - fn test_diseq_1() { - with_globals(|| { - let test_res = string_to_ts("(bar,baz)"); - let test_eqs = string_to_ts("bar,baz"); - assert_eq!(test_res == test_eqs, false) - }) - } - - #[test] - fn test_is_empty() { - with_globals(|| { - let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); - let test1: TokenStream = - TokenTree::Token(sp(0, 1), Token::Ident(Ident::from_str("a"), false)).into(); - let test2 = string_to_ts("foo(bar::baz)"); - - assert_eq!(test0.is_empty(), true); - assert_eq!(test1.is_empty(), false); - assert_eq!(test2.is_empty(), false); - }) - } - - #[test] - fn test_dotdotdot() { - let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::Token(sp(0, 1), Token::Dot).joint()); - builder.push(TokenTree::Token(sp(1, 2), Token::Dot).joint()); - builder.push(TokenTree::Token(sp(2, 3), Token::Dot)); - let stream = builder.build(); - assert!(stream.eq_unspanned(&string_to_ts("..."))); - assert_eq!(stream.trees().count(), 1); - } -} diff --git a/src/libsyntax/tokenstream/tests.rs b/src/libsyntax/tokenstream/tests.rs new file mode 100644 index 00000000000..72e22a49876 --- /dev/null +++ b/src/libsyntax/tokenstream/tests.rs @@ -0,0 +1,108 @@ +use super::*; + +use crate::ast::Name; +use crate::with_default_globals; +use crate::tests::string_to_stream; +use syntax_pos::{Span, BytePos, NO_EXPANSION}; + +fn string_to_ts(string: &str) -> TokenStream { + string_to_stream(string.to_owned()) +} + +fn sp(a: u32, b: u32) -> Span { + Span::new(BytePos(a), BytePos(b), NO_EXPANSION) +} + +#[test] +fn test_concat() { + with_default_globals(|| { + let test_res = string_to_ts("foo::bar::baz"); + let test_fst = string_to_ts("foo::bar"); + let test_snd = string_to_ts("::baz"); + let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + assert_eq!(test_res.trees().count(), 5); + assert_eq!(eq_res.trees().count(), 5); + assert_eq!(test_res.eq_unspanned(&eq_res), true); + }) +} + +#[test] +fn test_to_from_bijection() { + with_default_globals(|| { + let test_start = string_to_ts("foo::bar(baz)"); + let test_end = test_start.trees().collect(); + assert_eq!(test_start, test_end) + }) +} + +#[test] +fn test_eq_0() { + with_default_globals(|| { + let test_res = string_to_ts("foo"); + let test_eqs = string_to_ts("foo"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_1() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("::bar::baz"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_3() { + with_default_globals(|| { + let test_res = string_to_ts(""); + let test_eqs = string_to_ts(""); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_diseq_0() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("bar::baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_diseq_1() { + with_default_globals(|| { + let test_res = string_to_ts("(bar,baz)"); + let test_eqs = string_to_ts("bar,baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_is_empty() { + with_default_globals(|| { + let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); + let test1: TokenStream = + TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); + let test2 = string_to_ts("foo(bar::baz)"); + + assert_eq!(test0.is_empty(), true); + assert_eq!(test1.is_empty(), false); + assert_eq!(test2.is_empty(), false); + }) +} + +#[test] +fn test_dotdotdot() { + with_default_globals(|| { + let mut builder = TokenStreamBuilder::new(); + builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); + builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); + builder.push(TokenTree::token(token::Dot, sp(2, 3))); + let stream = builder.build(); + assert!(stream.eq_unspanned(&string_to_ts("..."))); + assert_eq!(stream.trees().count(), 1); + }) +} diff --git a/src/libsyntax/util/lev_distance.rs b/src/libsyntax/util/lev_distance.rs index 2f150d22159..4127a8c7fce 100644 --- a/src/libsyntax/util/lev_distance.rs +++ b/src/libsyntax/util/lev_distance.rs @@ -1,6 +1,9 @@ use std::cmp; use crate::symbol::Symbol; +#[cfg(test)] +mod tests; + /// Finds the Levenshtein distance between two strings pub fn lev_distance(a: &str, b: &str) -> usize { // cases which don't require further computation @@ -77,60 +80,3 @@ pub fn find_best_match_for_name<'a, T>(iter_names: T, if let Some((candidate, _)) = levenstein_match { Some(candidate) } else { None } } } - -#[test] -fn test_lev_distance() { - use std::char::{from_u32, MAX}; - // Test bytelength agnosticity - for c in (0..MAX as u32) - .filter_map(|i| from_u32(i)) - .map(|i| i.to_string()) { - assert_eq!(lev_distance(&c[..], &c[..]), 0); - } - - let a = "\nMäry häd ä little lämb\n\nLittle lämb\n"; - let b = "\nMary häd ä little lämb\n\nLittle lämb\n"; - let c = "Mary häd ä little lämb\n\nLittle lämb\n"; - assert_eq!(lev_distance(a, b), 1); - assert_eq!(lev_distance(b, a), 1); - assert_eq!(lev_distance(a, c), 2); - assert_eq!(lev_distance(c, a), 2); - assert_eq!(lev_distance(b, c), 1); - assert_eq!(lev_distance(c, b), 1); -} - -#[test] -fn test_find_best_match_for_name() { - use crate::with_globals; - with_globals(|| { - let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")]; - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", None), - Some(Symbol::intern("aaab")) - ); - - assert_eq!( - find_best_match_for_name(input.iter(), "1111111111", None), - None - ); - - let input = vec![Symbol::intern("aAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), "AAAA", None), - Some(Symbol::intern("aAAA")) - ); - - let input = vec![Symbol::intern("AAAA")]; - // Returns None because `lev_distance > max_dist / 3` - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", None), - None - ); - - let input = vec![Symbol::intern("AAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", Some(4)), - Some(Symbol::intern("AAAA")) - ); - }) -} diff --git a/src/libsyntax/util/lev_distance/tests.rs b/src/libsyntax/util/lev_distance/tests.rs new file mode 100644 index 00000000000..1a746a67ec0 --- /dev/null +++ b/src/libsyntax/util/lev_distance/tests.rs @@ -0,0 +1,58 @@ +use super::*; + +#[test] +fn test_lev_distance() { + use std::char::{from_u32, MAX}; + // Test bytelength agnosticity + for c in (0..MAX as u32) + .filter_map(|i| from_u32(i)) + .map(|i| i.to_string()) { + assert_eq!(lev_distance(&c[..], &c[..]), 0); + } + + let a = "\nMäry häd ä little lämb\n\nLittle lämb\n"; + let b = "\nMary häd ä little lämb\n\nLittle lämb\n"; + let c = "Mary häd ä little lämb\n\nLittle lämb\n"; + assert_eq!(lev_distance(a, b), 1); + assert_eq!(lev_distance(b, a), 1); + assert_eq!(lev_distance(a, c), 2); + assert_eq!(lev_distance(c, a), 2); + assert_eq!(lev_distance(b, c), 1); + assert_eq!(lev_distance(c, b), 1); +} + +#[test] +fn test_find_best_match_for_name() { + use crate::with_default_globals; + with_default_globals(|| { + let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")]; + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", None), + Some(Symbol::intern("aaab")) + ); + + assert_eq!( + find_best_match_for_name(input.iter(), "1111111111", None), + None + ); + + let input = vec![Symbol::intern("aAAA")]; + assert_eq!( + find_best_match_for_name(input.iter(), "AAAA", None), + Some(Symbol::intern("aAAA")) + ); + + let input = vec![Symbol::intern("AAAA")]; + // Returns None because `lev_distance > max_dist / 3` + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", None), + None + ); + + let input = vec![Symbol::intern("AAAA")]; + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", Some(4)), + Some(Symbol::intern("AAAA")) + ); + }) +} diff --git a/src/libsyntax/util/node_count.rs b/src/libsyntax/util/node_count.rs index 521edac8f5f..f17eb3b3943 100644 --- a/src/libsyntax/util/node_count.rs +++ b/src/libsyntax/util/node_count.rs @@ -131,9 +131,9 @@ impl<'ast> Visitor<'ast> for NodeCounter { self.count += 1; walk_generic_args(self, path_span, generic_args) } - fn visit_assoc_type_binding(&mut self, type_binding: &TypeBinding) { + fn visit_assoc_ty_constraint(&mut self, constraint: &AssocTyConstraint) { self.count += 1; - walk_assoc_type_binding(self, type_binding) + walk_assoc_ty_constraint(self, constraint) } fn visit_attribute(&mut self, _attr: &Attribute) { self.count += 1; diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 5f15ede7b0b..d71358f45c4 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,5 +1,5 @@ -use crate::parse::token::{Token, BinOpToken}; -use crate::symbol::keywords; +use crate::parse::token::{self, Token, BinOpToken}; +use crate::symbol::kw; use crate::ast::{self, BinOpKind}; /// Associative operator with precedence. @@ -45,8 +45,6 @@ pub enum AssocOp { GreaterEqual, /// `=` Assign, - /// `<-` - ObsoleteInPlace, /// `?=` where ? is one of the BinOpToken AssignOp(BinOpToken), /// `as` @@ -73,34 +71,33 @@ impl AssocOp { /// Creates a new AssocOP from a token pub fn from_token(t: &Token) -> Option<AssocOp> { use AssocOp::*; - match *t { - Token::BinOpEq(k) => Some(AssignOp(k)), - Token::LArrow => Some(ObsoleteInPlace), - Token::Eq => Some(Assign), - Token::BinOp(BinOpToken::Star) => Some(Multiply), - Token::BinOp(BinOpToken::Slash) => Some(Divide), - Token::BinOp(BinOpToken::Percent) => Some(Modulus), - Token::BinOp(BinOpToken::Plus) => Some(Add), - Token::BinOp(BinOpToken::Minus) => Some(Subtract), - Token::BinOp(BinOpToken::Shl) => Some(ShiftLeft), - Token::BinOp(BinOpToken::Shr) => Some(ShiftRight), - Token::BinOp(BinOpToken::And) => Some(BitAnd), - Token::BinOp(BinOpToken::Caret) => Some(BitXor), - Token::BinOp(BinOpToken::Or) => Some(BitOr), - Token::Lt => Some(Less), - Token::Le => Some(LessEqual), - Token::Ge => Some(GreaterEqual), - Token::Gt => Some(Greater), - Token::EqEq => Some(Equal), - Token::Ne => Some(NotEqual), - Token::AndAnd => Some(LAnd), - Token::OrOr => Some(LOr), - Token::DotDot => Some(DotDot), - Token::DotDotEq => Some(DotDotEq), + match t.kind { + token::BinOpEq(k) => Some(AssignOp(k)), + token::Eq => Some(Assign), + token::BinOp(BinOpToken::Star) => Some(Multiply), + token::BinOp(BinOpToken::Slash) => Some(Divide), + token::BinOp(BinOpToken::Percent) => Some(Modulus), + token::BinOp(BinOpToken::Plus) => Some(Add), + token::BinOp(BinOpToken::Minus) => Some(Subtract), + token::BinOp(BinOpToken::Shl) => Some(ShiftLeft), + token::BinOp(BinOpToken::Shr) => Some(ShiftRight), + token::BinOp(BinOpToken::And) => Some(BitAnd), + token::BinOp(BinOpToken::Caret) => Some(BitXor), + token::BinOp(BinOpToken::Or) => Some(BitOr), + token::Lt => Some(Less), + token::Le => Some(LessEqual), + token::Ge => Some(GreaterEqual), + token::Gt => Some(Greater), + token::EqEq => Some(Equal), + token::Ne => Some(NotEqual), + token::AndAnd => Some(LAnd), + token::OrOr => Some(LOr), + token::DotDot => Some(DotDot), + token::DotDotEq => Some(DotDotEq), // DotDotDot is no longer supported, but we need some way to display the error - Token::DotDotDot => Some(DotDotEq), - Token::Colon => Some(Colon), - _ if t.is_keyword(keywords::As) => Some(As), + token::DotDotDot => Some(DotDotEq), + token::Colon => Some(Colon), + _ if t.is_keyword(kw::As) => Some(As), _ => None } } @@ -145,7 +142,6 @@ impl AssocOp { LAnd => 6, LOr => 5, DotDot | DotDotEq => 4, - ObsoleteInPlace => 3, Assign | AssignOp(_) => 2, } } @@ -155,7 +151,7 @@ impl AssocOp { use AssocOp::*; // NOTE: it is a bug to have an operators that has same precedence but different fixities! match *self { - ObsoleteInPlace | Assign | AssignOp(_) => Fixity::Right, + Assign | AssignOp(_) => Fixity::Right, As | Multiply | Divide | Modulus | Add | Subtract | ShiftLeft | ShiftRight | BitAnd | BitXor | BitOr | Less | Greater | LessEqual | GreaterEqual | Equal | NotEqual | LAnd | LOr | Colon => Fixity::Left, @@ -167,7 +163,7 @@ impl AssocOp { use AssocOp::*; match *self { Less | Greater | LessEqual | GreaterEqual | Equal | NotEqual => true, - ObsoleteInPlace | Assign | AssignOp(_) | As | Multiply | Divide | Modulus | Add | + Assign | AssignOp(_) | As | Multiply | Divide | Modulus | Add | Subtract | ShiftLeft | ShiftRight | BitAnd | BitXor | BitOr | LAnd | LOr | DotDot | DotDotEq | Colon => false } @@ -176,7 +172,7 @@ impl AssocOp { pub fn is_assign_like(&self) -> bool { use AssocOp::*; match *self { - Assign | AssignOp(_) | ObsoleteInPlace => true, + Assign | AssignOp(_) => true, Less | Greater | LessEqual | GreaterEqual | Equal | NotEqual | As | Multiply | Divide | Modulus | Add | Subtract | ShiftLeft | ShiftRight | BitAnd | BitXor | BitOr | LAnd | LOr | DotDot | DotDotEq | Colon => false @@ -204,7 +200,32 @@ impl AssocOp { BitOr => Some(BinOpKind::BitOr), LAnd => Some(BinOpKind::And), LOr => Some(BinOpKind::Or), - ObsoleteInPlace | Assign | AssignOp(_) | As | DotDot | DotDotEq | Colon => None + Assign | AssignOp(_) | As | DotDot | DotDotEq | Colon => None + } + } + + /// This operator could be used to follow a block unambiguously. + /// + /// This is used for error recovery at the moment, providing a suggestion to wrap blocks with + /// parentheses while having a high degree of confidence on the correctness of the suggestion. + pub fn can_continue_expr_unambiguously(&self) -> bool { + use AssocOp::*; + match self { + BitXor | // `{ 42 } ^ 3` + Assign | // `{ 42 } = { 42 }` + Divide | // `{ 42 } / 42` + Modulus | // `{ 42 } % 2` + ShiftRight | // `{ 42 } >> 2` + LessEqual | // `{ 42 } <= 3` + Greater | // `{ 42 } > 3` + GreaterEqual | // `{ 42 } >= 3` + AssignOp(_) | // `{ 42 } +=` + LAnd | // `{ 42 } &&foo` + As | // `{ 42 } as usize` + // Equal | // `{ 42 } == { 42 }` Accepting these here would regress incorrect + // NotEqual | // `{ 42 } != { 42 } struct literals parser recovery. + Colon => true, // `{ 42 }: usize` + _ => false, } } } @@ -213,7 +234,7 @@ pub const PREC_RESET: i8 = -100; pub const PREC_CLOSURE: i8 = -40; pub const PREC_JUMP: i8 = -30; pub const PREC_RANGE: i8 = -10; -// The range 2 ... 14 is reserved for AssocOp binary operator precedences. +// The range 2..=14 is reserved for AssocOp binary operator precedences. pub const PREC_PREFIX: i8 = 50; pub const PREC_POSTFIX: i8 = 60; pub const PREC_PAREN: i8 = 99; @@ -231,7 +252,6 @@ pub enum ExprPrecedence { Binary(BinOpKind), - ObsoleteInPlace, Cast, Type, @@ -240,6 +260,7 @@ pub enum ExprPrecedence { Box, AddrOf, + Let, Unary, Call, @@ -257,9 +278,7 @@ pub enum ExprPrecedence { Path, Paren, If, - IfLet, While, - WhileLet, ForLoop, Loop, Match, @@ -267,6 +286,7 @@ pub enum ExprPrecedence { TryBlock, Struct, Async, + Await, Err, } @@ -288,7 +308,6 @@ impl ExprPrecedence { // Binop-like expr kinds, handled by `AssocOp`. ExprPrecedence::Binary(op) => AssocOp::from_ast_binop(op).precedence() as i8, - ExprPrecedence::ObsoleteInPlace => AssocOp::ObsoleteInPlace.precedence() as i8, ExprPrecedence::Cast => AssocOp::As.precedence() as i8, ExprPrecedence::Type => AssocOp::Colon.precedence() as i8, @@ -298,9 +317,15 @@ impl ExprPrecedence { // Unary, prefix ExprPrecedence::Box | ExprPrecedence::AddrOf | + // Here `let pats = expr` has `let pats =` as a "unary" prefix of `expr`. + // However, this is not exactly right. When `let _ = a` is the LHS of a binop we + // need parens sometimes. E.g. we can print `(let _ = a) && b` as `let _ = a && b` + // but we need to print `(let _ = a) < b` as-is with parens. + ExprPrecedence::Let | ExprPrecedence::Unary => PREC_PREFIX, // Unary, postfix + ExprPrecedence::Await | ExprPrecedence::Call | ExprPrecedence::MethodCall | ExprPrecedence::Field | @@ -317,9 +342,7 @@ impl ExprPrecedence { ExprPrecedence::Path | ExprPrecedence::Paren | ExprPrecedence::If | - ExprPrecedence::IfLet | ExprPrecedence::While | - ExprPrecedence::WhileLet | ExprPrecedence::ForLoop | ExprPrecedence::Loop | ExprPrecedence::Match | @@ -332,6 +355,19 @@ impl ExprPrecedence { } } +/// In `let p = e`, operators with precedence `<=` this one requires parenthesis in `e`. +crate fn prec_let_scrutinee_needs_par() -> usize { + AssocOp::LAnd.precedence() +} + +/// Suppose we have `let _ = e` and the `order` of `e`. +/// Is the `order` such that `e` in `let _ = e` needs parenthesis when it is on the RHS? +/// +/// Conversely, suppose that we have `(let _ = a) OP b` and `order` is that of `OP`. +/// Can we print this as `let _ = a OP b`? +crate fn needs_par_as_let_scrutinee(order: i8) -> bool { + order <= prec_let_scrutinee_needs_par() as i8 +} /// Expressions that syntactically contain an "exterior" struct literal i.e., not surrounded by any /// parens or other delimiters, e.g., `X { y: 1 }`, `X { y: 1 }.method()`, `foo == X { y: 1 }` and @@ -346,6 +382,7 @@ pub fn contains_exterior_struct_lit(value: &ast::Expr) -> bool { // X { y: 1 } + X { y: 2 } contains_exterior_struct_lit(&lhs) || contains_exterior_struct_lit(&rhs) } + ast::ExprKind::Await(ref x) | ast::ExprKind::Unary(_, ref x) | ast::ExprKind::Cast(ref x, _) | ast::ExprKind::Type(ref x, _) | diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs deleted file mode 100644 index 733c4f83e37..00000000000 --- a/src/libsyntax/util/parser_testing.rs +++ /dev/null @@ -1,160 +0,0 @@ -use crate::ast::{self, Ident}; -use crate::source_map::FilePathMapping; -use crate::parse::{ParseSess, PResult, source_file_to_stream}; -use crate::parse::{lexer, new_parser_from_source_str}; -use crate::parse::parser::Parser; -use crate::ptr::P; -use crate::tokenstream::TokenStream; - -use std::iter::Peekable; -use std::path::PathBuf; - -/// Map a string to tts, using a made-up filename: -pub fn string_to_stream(source_str: String) -> TokenStream { - let ps = ParseSess::new(FilePathMapping::empty()); - source_file_to_stream( - &ps, - ps.source_map().new_source_file(PathBuf::from("bogofile").into(), - source_str, - ), None).0 -} - -/// Map string to parser (via tts) -pub fn string_to_parser<'a>(ps: &'a ParseSess, source_str: String) -> Parser<'a> { - new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) -} - -fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where - F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, -{ - let mut p = string_to_parser(&ps, s); - let x = panictry!(f(&mut p)); - p.sess.span_diagnostic.abort_if_errors(); - x -} - -/// Parse a string, return a crate. -pub fn string_to_crate (source_str : String) -> ast::Crate { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_crate_mod() - }) -} - -/// Parse a string, return an expr -pub fn string_to_expr (source_str : String) -> P<ast::Expr> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_expr() - }) -} - -/// Parse a string, return an item -pub fn string_to_item (source_str : String) -> Option<P<ast::Item>> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_item() - }) -} - -/// Parse a string, return a pat. Uses "irrefutable"... which doesn't -/// (currently) affect parsing. -pub fn string_to_pat(source_str: String) -> P<ast::Pat> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_pat(None) - }) -} - -/// Converts a vector of strings to a vector of Ident's -pub fn strs_to_idents(ids: Vec<&str> ) -> Vec<Ident> { - ids.iter().map(|u| Ident::from_str(*u)).collect() -} - -/// Does the given string match the pattern? whitespace in the first string -/// may be deleted or replaced with other whitespace to match the pattern. -/// This function is relatively Unicode-ignorant; fortunately, the careful design -/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). -pub fn matches_codepattern(a : &str, b : &str) -> bool { - let mut a_iter = a.chars().peekable(); - let mut b_iter = b.chars().peekable(); - - loop { - let (a, b) = match (a_iter.peek(), b_iter.peek()) { - (None, None) => return true, - (None, _) => return false, - (Some(&a), None) => { - if is_pattern_whitespace(a) { - break // trailing whitespace check is out of loop for borrowck - } else { - return false - } - } - (Some(&a), Some(&b)) => (a, b) - }; - - if is_pattern_whitespace(a) && is_pattern_whitespace(b) { - // skip whitespace for a and b - scan_for_non_ws_or_end(&mut a_iter); - scan_for_non_ws_or_end(&mut b_iter); - } else if is_pattern_whitespace(a) { - // skip whitespace for a - scan_for_non_ws_or_end(&mut a_iter); - } else if a == b { - a_iter.next(); - b_iter.next(); - } else { - return false - } - } - - // check if a has *only* trailing whitespace - a_iter.all(is_pattern_whitespace) -} - -/// Advances the given peekable `Iterator` until it reaches a non-whitespace character -fn scan_for_non_ws_or_end<I: Iterator<Item= char>>(iter: &mut Peekable<I>) { - while lexer::is_pattern_whitespace(iter.peek().cloned()) { - iter.next(); - } -} - -pub fn is_pattern_whitespace(c: char) -> bool { - lexer::is_pattern_whitespace(Some(c)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn eqmodws() { - assert_eq!(matches_codepattern("",""),true); - assert_eq!(matches_codepattern("","a"),false); - assert_eq!(matches_codepattern("a",""),false); - assert_eq!(matches_codepattern("a","a"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); - assert_eq!(matches_codepattern("a b","a b"),true); - assert_eq!(matches_codepattern("ab","a b"),false); - assert_eq!(matches_codepattern("a b","ab"),true); - assert_eq!(matches_codepattern(" a b","ab"),true); - } - - #[test] - fn pattern_whitespace() { - assert_eq!(matches_codepattern("","\x0C"), false); - assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); - } - - #[test] - fn non_pattern_whitespace() { - // These have the property 'White_Space' but not 'Pattern_White_Space' - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); - assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); - } -} diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 6b3a30ccb54..596c5b46b98 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -31,6 +31,16 @@ pub enum FnKind<'a> { Closure(&'a Expr), } +impl<'a> FnKind<'a> { + pub fn header(&self) -> Option<&'a FnHeader> { + match *self { + FnKind::ItemFn(_, header, _, _) => Some(header), + FnKind::Method(_, sig, _, _) => Some(&sig.header), + FnKind::Closure(_) => None, + } + } +} + /// Each method of the Visitor trait is a hook to be potentially /// overridden. Each method's default implementation recursively visits /// the substructure of the input via the corresponding `walk` method; @@ -56,6 +66,7 @@ pub trait Visitor<'ast>: Sized { fn visit_local(&mut self, l: &'ast Local) { walk_local(self, l) } fn visit_block(&mut self, b: &'ast Block) { walk_block(self, b) } fn visit_stmt(&mut self, s: &'ast Stmt) { walk_stmt(self, s) } + fn visit_arg(&mut self, arg: &'ast Arg) { walk_arg(self, arg) } fn visit_arm(&mut self, a: &'ast Arm) { walk_arm(self, a) } fn visit_pat(&mut self, p: &'ast Pat) { walk_pat(self, p) } fn visit_anon_const(&mut self, c: &'ast AnonConst) { walk_anon_const(self, c) } @@ -129,8 +140,8 @@ pub trait Visitor<'ast>: Sized { GenericArg::Const(ct) => self.visit_anon_const(ct), } } - fn visit_assoc_type_binding(&mut self, type_binding: &'ast TypeBinding) { - walk_assoc_type_binding(self, type_binding) + fn visit_assoc_ty_constraint(&mut self, constraint: &'ast AssocTyConstraint) { + walk_assoc_ty_constraint(self, constraint) } fn visit_attribute(&mut self, attr: &'ast Attribute) { walk_attribute(self, attr) @@ -244,24 +255,24 @@ pub fn walk_item<'a, V: Visitor<'a>>(visitor: &mut V, item: &'a Item) { walk_list!(visitor, visit_foreign_item, &foreign_module.items); } ItemKind::GlobalAsm(ref ga) => visitor.visit_global_asm(ga), - ItemKind::Ty(ref typ, ref type_parameters) => { + ItemKind::TyAlias(ref typ, ref generics) => { visitor.visit_ty(typ); - visitor.visit_generics(type_parameters) + visitor.visit_generics(generics) } - ItemKind::Existential(ref bounds, ref type_parameters) => { + ItemKind::OpaqueTy(ref bounds, ref generics) => { walk_list!(visitor, visit_param_bound, bounds); - visitor.visit_generics(type_parameters) + visitor.visit_generics(generics) } - ItemKind::Enum(ref enum_definition, ref type_parameters) => { - visitor.visit_generics(type_parameters); - visitor.visit_enum_def(enum_definition, type_parameters, item.id, item.span) + ItemKind::Enum(ref enum_definition, ref generics) => { + visitor.visit_generics(generics); + visitor.visit_enum_def(enum_definition, generics, item.id, item.span) } ItemKind::Impl(_, _, _, - ref type_parameters, + ref generics, ref opt_trait_reference, ref typ, ref impl_items) => { - visitor.visit_generics(type_parameters); + visitor.visit_generics(generics); walk_list!(visitor, visit_trait_ref, opt_trait_reference); visitor.visit_ty(typ); walk_list!(visitor, visit_impl_item, impl_items); @@ -394,7 +405,7 @@ pub fn walk_generic_args<'a, V>(visitor: &mut V, match *generic_args { GenericArgs::AngleBracketed(ref data) => { walk_list!(visitor, visit_generic_arg, &data.args); - walk_list!(visitor, visit_assoc_type_binding, &data.bindings); + walk_list!(visitor, visit_assoc_ty_constraint, &data.constraints); } GenericArgs::Parenthesized(ref data) => { walk_list!(visitor, visit_ty, &data.inputs); @@ -403,17 +414,24 @@ pub fn walk_generic_args<'a, V>(visitor: &mut V, } } -pub fn walk_assoc_type_binding<'a, V: Visitor<'a>>(visitor: &mut V, - type_binding: &'a TypeBinding) { - visitor.visit_ident(type_binding.ident); - visitor.visit_ty(&type_binding.ty); +pub fn walk_assoc_ty_constraint<'a, V: Visitor<'a>>(visitor: &mut V, + constraint: &'a AssocTyConstraint) { + visitor.visit_ident(constraint.ident); + match constraint.kind { + AssocTyConstraintKind::Equality { ref ty } => { + visitor.visit_ty(ty); + } + AssocTyConstraintKind::Bound { ref bounds } => { + walk_list!(visitor, visit_param_bound, bounds); + } + } } pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) { match pattern.node { - PatKind::TupleStruct(ref path, ref children, _) => { + PatKind::TupleStruct(ref path, ref elems) => { visitor.visit_path(path, pattern.id); - walk_list!(visitor, visit_pat, children); + walk_list!(visitor, visit_pat, elems); } PatKind::Path(ref opt_qself, ref path) => { if let Some(ref qself) = *opt_qself { @@ -429,8 +447,8 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) { visitor.visit_pat(&field.node.pat) } } - PatKind::Tuple(ref tuple_elements, _) => { - walk_list!(visitor, visit_pat, tuple_elements); + PatKind::Tuple(ref elems) => { + walk_list!(visitor, visit_pat, elems); } PatKind::Box(ref subpattern) | PatKind::Ref(ref subpattern, _) | @@ -446,11 +464,9 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) { visitor.visit_expr(lower_bound); visitor.visit_expr(upper_bound); } - PatKind::Wild => (), - PatKind::Slice(ref prepatterns, ref slice_pattern, ref postpatterns) => { - walk_list!(visitor, visit_pat, prepatterns); - walk_list!(visitor, visit_pat, slice_pattern); - walk_list!(visitor, visit_pat, postpatterns); + PatKind::Wild | PatKind::Rest => {}, + PatKind::Slice(ref elems) => { + walk_list!(visitor, visit_pat, elems); } PatKind::Mac(ref mac) => visitor.visit_mac(mac), } @@ -489,7 +505,7 @@ pub fn walk_generic_param<'a, V: Visitor<'a>>(visitor: &mut V, param: &'a Generi walk_list!(visitor, visit_attribute, param.attrs.iter()); walk_list!(visitor, visit_param_bound, ¶m.bounds); match param.kind { - GenericParamKind::Lifetime => {} + GenericParamKind::Lifetime => (), GenericParamKind::Type { ref default } => walk_list!(visitor, visit_ty, default), GenericParamKind::Const { ref ty, .. } => visitor.visit_ty(ty), } @@ -532,11 +548,10 @@ pub fn walk_fn_ret_ty<'a, V: Visitor<'a>>(visitor: &mut V, ret_ty: &'a FunctionR } pub fn walk_fn_decl<'a, V: Visitor<'a>>(visitor: &mut V, function_declaration: &'a FnDecl) { - for argument in &function_declaration.inputs { - visitor.visit_pat(&argument.pat); - visitor.visit_ty(&argument.ty) + for arg in &function_declaration.inputs { + visitor.visit_arg(arg); } - visitor.visit_fn_ret_ty(&function_declaration.output) + visitor.visit_fn_ret_ty(&function_declaration.output); } pub fn walk_fn<'a, V>(visitor: &mut V, kind: FnKind<'a>, declaration: &'a FnDecl, _span: Span) @@ -601,10 +616,10 @@ pub fn walk_impl_item<'a, V: Visitor<'a>>(visitor: &mut V, impl_item: &'a ImplIt visitor.visit_fn(FnKind::Method(impl_item.ident, sig, Some(&impl_item.vis), body), &sig.decl, impl_item.span, impl_item.id); } - ImplItemKind::Type(ref ty) => { + ImplItemKind::TyAlias(ref ty) => { visitor.visit_ty(ty); } - ImplItemKind::Existential(ref bounds) => { + ImplItemKind::OpaqueTy(ref bounds) => { walk_list!(visitor, visit_param_bound, bounds); } ImplItemKind::Macro(ref mac) => { @@ -647,8 +662,8 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) { } } -pub fn walk_mac<'a, V: Visitor<'a>>(_: &mut V, _: &Mac) { - // Empty! +pub fn walk_mac<'a, V: Visitor<'a>>(visitor: &mut V, mac: &'a Mac) { + visitor.visit_path(&mac.node.path, DUMMY_NODE_ID); } pub fn walk_anon_const<'a, V: Visitor<'a>>(visitor: &mut V, constant: &'a AnonConst) { @@ -663,10 +678,6 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { ExprKind::Box(ref subexpression) => { visitor.visit_expr(subexpression) } - ExprKind::ObsoleteInPlace(ref place, ref subexpression) => { - visitor.visit_expr(place); - visitor.visit_expr(subexpression) - } ExprKind::Array(ref subexpressions) => { walk_list!(visitor, visit_expr, subexpressions); } @@ -701,11 +712,14 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { ExprKind::AddrOf(_, ref subexpression) | ExprKind::Unary(_, ref subexpression) => { visitor.visit_expr(subexpression) } - ExprKind::Lit(_) => {} ExprKind::Cast(ref subexpression, ref typ) | ExprKind::Type(ref subexpression, ref typ) => { visitor.visit_expr(subexpression); visitor.visit_ty(typ) } + ExprKind::Let(ref pats, ref scrutinee) => { + walk_list!(visitor, visit_pat, pats); + visitor.visit_expr(scrutinee); + } ExprKind::If(ref head_expression, ref if_block, ref optional_else) => { visitor.visit_expr(head_expression); visitor.visit_block(if_block); @@ -716,18 +730,6 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { visitor.visit_expr(subexpression); visitor.visit_block(block); } - ExprKind::IfLet(ref pats, ref subexpression, ref if_block, ref optional_else) => { - walk_list!(visitor, visit_pat, pats); - visitor.visit_expr(subexpression); - visitor.visit_block(if_block); - walk_list!(visitor, visit_expr, optional_else); - } - ExprKind::WhileLet(ref pats, ref subexpression, ref block, ref opt_label) => { - walk_list!(visitor, visit_label, opt_label); - walk_list!(visitor, visit_pat, pats); - visitor.visit_expr(subexpression); - visitor.visit_block(block); - } ExprKind::ForLoop(ref pattern, ref subexpression, ref block, ref opt_label) => { walk_list!(visitor, visit_label, opt_label); visitor.visit_pat(pattern); @@ -755,6 +757,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { ExprKind::Async(_, _, ref body) => { visitor.visit_block(body); } + ExprKind::Await(ref expr) => visitor.visit_expr(expr), ExprKind::Assign(ref left_hand_expression, ref right_hand_expression) => { visitor.visit_expr(left_hand_expression); visitor.visit_expr(right_hand_expression); @@ -812,18 +815,22 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) { ExprKind::TryBlock(ref body) => { visitor.visit_block(body) } - ExprKind::Err => {} + ExprKind::Lit(_) | ExprKind::Err => {} } visitor.visit_expr_post(expression) } +pub fn walk_arg<'a, V: Visitor<'a>>(visitor: &mut V, arg: &'a Arg) { + walk_list!(visitor, visit_attribute, arg.attrs.iter()); + visitor.visit_pat(&arg.pat); + visitor.visit_ty(&arg.ty); +} + pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) { walk_list!(visitor, visit_pat, &arm.pats); - if let Some(ref g) = &arm.guard { - match g { - Guard::If(ref e) => visitor.visit_expr(e), - } + if let Some(ref e) = &arm.guard { + visitor.visit_expr(e); } visitor.visit_expr(&arm.body); walk_list!(visitor, visit_attribute, &arm.attrs); @@ -841,7 +848,7 @@ pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) pub fn walk_tt<'a, V: Visitor<'a>>(visitor: &mut V, tt: TokenTree) { match tt { - TokenTree::Token(_, tok) => visitor.visit_token(tok), + TokenTree::Token(token) => visitor.visit_token(token), TokenTree::Delimited(_, _, tts) => visitor.visit_tts(tts), } } |