From eea6f23a0ed67fd8c6b8e1b02cda3628fee56b2f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 24 Oct 2019 06:33:12 +1100 Subject: Make doc comments cheaper with `AttrKind`. `AttrKind` is a new type with two variants, `Normal` and `DocComment`. It's a big performance win (over 10% in some cases) because `DocComment` lets doc comments (which are common) be represented very cheaply. `Attribute` gets some new helper methods to ease the transition: - `has_name()`: check if the attribute name matches a single `Symbol`; for `DocComment` variants it succeeds if the symbol is `sym::doc`. - `is_doc_comment()`: check if it has a `DocComment` kind. - `{get,unwrap}_normal_item()`: extract the item from a `Normal` variant; panic otherwise. Fixes #60935. --- src/libsyntax/parse/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libsyntax/parse/tests.rs') diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 3bdb9227b4e..169eb954efa 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -246,7 +246,7 @@ let mut fflags: c_int = wb(); let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); let item = parse_item_from_source_str(name_2, source, &sess) .unwrap().unwrap(); - let docs = item.attrs.iter().filter(|a| a.path == sym::doc) + let docs = item.attrs.iter().filter(|a| a.has_name(sym::doc)) .map(|a| a.value_str().unwrap().to_string()).collect::>(); let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; assert_eq!(&docs[..], b); -- cgit 1.4.1-3-g733a5 From 8fa8e02c282f6dc4ebcd0f8403a881116a948443 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 12:29:23 +0200 Subject: syntax: simplify imports --- src/libsyntax/parse/lexer/comments.rs | 5 +++-- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/tests.rs | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) (limited to 'src/libsyntax/parse/tests.rs') diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index ac79ce323bf..33415bdcb62 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -1,9 +1,10 @@ pub use CommentStyle::*; +use super::is_block_doc_comment; + use crate::ast; use crate::source_map::SourceMap; -use crate::parse::lexer::is_block_doc_comment; -use crate::parse::lexer::ParseSess; +use crate::sess::ParseSess; use syntax_pos::{BytePos, CharPos, Pos, FileName}; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index cc8738edff7..fcb7c4d1323 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -5,7 +5,7 @@ use crate::ptr::P; use crate::ast::{self, Attribute, Pat, PatKind, FieldPat, RangeEnd, RangeSyntax, Mac}; use crate::ast::{BindingMode, Ident, Mutability, Path, QSelf, Expr, ExprKind}; use crate::mut_visit::{noop_visit_pat, noop_visit_mac, MutVisitor}; -use crate::parse::token::{self}; +use crate::parse::token; use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 169eb954efa..201eaca947b 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -2,8 +2,8 @@ use super::*; use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; -use crate::parse::{ParseSess, PResult}; -use crate::parse::new_parser_from_source_str; +use crate::sess::ParseSess; +use crate::parse::{PResult, new_parser_from_source_str}; use crate::parse::token::Token; use crate::print::pprust::item_to_string; use crate::ptr::P; -- cgit 1.4.1-3-g733a5 From 9d6768a478b8a6afa1e16dfebe9d79bd3f508cdf Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 11 Oct 2019 12:46:32 +0200 Subject: syntax::parser::token -> syntax::token --- src/librustc/hir/lowering.rs | 2 +- src/librustc/hir/map/def_collector.rs | 2 +- src/librustc/ich/impls_syntax.rs | 4 +- src/librustc_interface/interface.rs | 2 +- src/librustc_resolve/build_reduced_graph.rs | 2 +- src/librustc_save_analysis/dump_visitor.rs | 2 +- src/librustc_save_analysis/span_utils.rs | 2 +- src/librustdoc/html/highlight.rs | 2 +- src/librustdoc/passes/check_code_block_syntax.rs | 2 +- src/libsyntax/ast.rs | 2 +- src/libsyntax/attr/mod.rs | 2 +- src/libsyntax/feature_gate/check.rs | 2 +- src/libsyntax/lib.rs | 1 + src/libsyntax/mut_visit.rs | 2 +- src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/lexer/tests.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 2 +- src/libsyntax/parse/lexer/unicode_chars.rs | 2 +- src/libsyntax/parse/literal.rs | 2 +- src/libsyntax/parse/mod.rs | 3 +- src/libsyntax/parse/parser/attr.rs | 2 +- src/libsyntax/parse/parser/diagnostics.rs | 2 +- src/libsyntax/parse/parser/expr.rs | 2 +- src/libsyntax/parse/parser/generics.rs | 2 +- src/libsyntax/parse/parser/mod.rs | 2 +- src/libsyntax/parse/parser/module.rs | 2 +- src/libsyntax/parse/parser/pat.rs | 2 +- src/libsyntax/parse/parser/path.rs | 2 +- src/libsyntax/parse/parser/stmt.rs | 2 +- src/libsyntax/parse/parser/ty.rs | 2 +- src/libsyntax/parse/tests.rs | 2 +- src/libsyntax/parse/token.rs | 728 ----------------------- src/libsyntax/print/pprust.rs | 2 +- src/libsyntax/token.rs | 728 +++++++++++++++++++++++ src/libsyntax/tokenstream.rs | 2 +- src/libsyntax/util/parser.rs | 2 +- src/libsyntax/visit.rs | 2 +- src/libsyntax_expand/base.rs | 2 +- src/libsyntax_expand/expand.rs | 2 +- src/libsyntax_expand/mbe.rs | 2 +- src/libsyntax_expand/mbe/macro_check.rs | 2 +- src/libsyntax_expand/mbe/macro_parser.rs | 2 +- src/libsyntax_expand/mbe/macro_rules.rs | 3 +- src/libsyntax_expand/mbe/quoted.rs | 2 +- src/libsyntax_expand/mbe/transcribe.rs | 2 +- src/libsyntax_expand/proc_macro.rs | 3 +- src/libsyntax_expand/proc_macro_server.rs | 7 +- src/libsyntax_ext/asm.rs | 2 +- src/libsyntax_ext/assert.rs | 2 +- src/libsyntax_ext/cfg.rs | 2 +- src/libsyntax_ext/cmdline_attrs.rs | 3 +- src/libsyntax_ext/concat_idents.rs | 2 +- src/libsyntax_ext/format.rs | 2 +- src/libsyntax_ext/global_asm.rs | 2 +- src/libsyntax_ext/plugin_macro_defs.rs | 4 +- src/libsyntax_ext/source_util.rs | 3 +- src/test/ui-fulldeps/ast_stmt_expr_attr.rs | 2 +- src/test/ui-fulldeps/auxiliary/roman-numerals.rs | 2 +- 58 files changed, 792 insertions(+), 789 deletions(-) delete mode 100644 src/libsyntax/parse/token.rs create mode 100644 src/libsyntax/token.rs (limited to 'src/libsyntax/parse/tests.rs') diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 6344c7a233c..87ad4ace592 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -66,7 +66,7 @@ use syntax::ptr::P as AstP; use syntax::ast::*; use syntax::errors; use syntax::print::pprust; -use syntax::parse::token::{self, Nonterminal, Token}; +use syntax::token::{self, Nonterminal, Token}; use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::sess::ParseSess; use syntax::source_map::{respan, ExpnData, ExpnKind, DesugaringKind, Spanned}; diff --git a/src/librustc/hir/map/def_collector.rs b/src/librustc/hir/map/def_collector.rs index e9970e30bf9..57c1421bde6 100644 --- a/src/librustc/hir/map/def_collector.rs +++ b/src/librustc/hir/map/def_collector.rs @@ -4,7 +4,7 @@ use crate::hir::def_id::DefIndex; use syntax::ast::*; use syntax::visit; use syntax::symbol::{kw, sym}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax_pos::hygiene::ExpnId; use syntax_pos::Span; diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 2201c4b0980..c401bd17dd4 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -8,9 +8,9 @@ use std::mem; use syntax::ast; use syntax::feature_gate; -use syntax::parse::token; -use syntax::symbol::SymbolStr; +use syntax::token; use syntax::tokenstream; +use syntax_pos::symbol::SymbolStr; use syntax_pos::SourceFile; use crate::hir::def_id::{DefId, CrateNum, CRATE_DEF_INDEX}; diff --git a/src/librustc_interface/interface.rs b/src/librustc_interface/interface.rs index 4e4d6d982fb..034e861b212 100644 --- a/src/librustc_interface/interface.rs +++ b/src/librustc_interface/interface.rs @@ -16,7 +16,7 @@ use std::result; use std::sync::{Arc, Mutex}; use syntax::{self, parse}; use syntax::ast::{self, MetaItemKind}; -use syntax::parse::token; +use syntax::token; use syntax::source_map::{FileName, FileLoader, SourceMap}; use syntax::sess::ParseSess; use syntax_pos::edition; diff --git a/src/librustc_resolve/build_reduced_graph.rs b/src/librustc_resolve/build_reduced_graph.rs index 55f054d0be3..4239518b879 100644 --- a/src/librustc_resolve/build_reduced_graph.rs +++ b/src/librustc_resolve/build_reduced_graph.rs @@ -33,7 +33,7 @@ use syntax::attr; use syntax::ast::{self, Block, ForeignItem, ForeignItemKind, Item, ItemKind, NodeId}; use syntax::ast::{MetaItemKind, StmtKind, TraitItem, TraitItemKind}; use syntax::feature_gate::is_builtin_attr; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::print::pprust; use syntax::{span_err, struct_span_err}; use syntax::source_map::{respan, Spanned}; diff --git a/src/librustc_save_analysis/dump_visitor.rs b/src/librustc_save_analysis/dump_visitor.rs index a372106d379..5c5fbcc07de 100644 --- a/src/librustc_save_analysis/dump_visitor.rs +++ b/src/librustc_save_analysis/dump_visitor.rs @@ -24,7 +24,7 @@ use std::path::Path; use std::env; use syntax::ast::{self, Attribute, NodeId, PatKind}; -use syntax::parse::token; +use syntax::token; use syntax::visit::{self, Visitor}; use syntax::print::pprust::{ bounds_to_string, diff --git a/src/librustc_save_analysis/span_utils.rs b/src/librustc_save_analysis/span_utils.rs index fb9919d777d..4d0780cf94d 100644 --- a/src/librustc_save_analysis/span_utils.rs +++ b/src/librustc_save_analysis/span_utils.rs @@ -3,7 +3,7 @@ use rustc::session::Session; use crate::generated_code; use syntax::parse::lexer::{self, StringReader}; -use syntax::parse::token::{self, TokenKind}; +use syntax::token::{self, TokenKind}; use syntax_pos::*; #[derive(Clone)] diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 88ba13f2796..4bd72f7e61c 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -13,7 +13,7 @@ use std::io::prelude::*; use syntax::source_map::SourceMap; use syntax::parse::lexer; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; use syntax_pos::{Span, FileName}; diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs index 10e15ab8881..4603e77b0fd 100644 --- a/src/librustdoc/passes/check_code_block_syntax.rs +++ b/src/librustdoc/passes/check_code_block_syntax.rs @@ -1,6 +1,6 @@ use errors::Applicability; use syntax::parse::lexer::{StringReader as Lexer}; -use syntax::parse::token; +use syntax::token; use syntax::sess::ParseSess; use syntax::source_map::FilePathMapping; use syntax_pos::{InnerSpan, FileName}; diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 67d1acbccfb..18151a1586c 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -24,9 +24,9 @@ pub use crate::util::parser::ExprPrecedence; pub use syntax_pos::symbol::{Ident, Symbol as Name}; -use crate::parse::token::{self, DelimToken}; use crate::ptr::P; use crate::source_map::{dummy_spanned, respan, Spanned}; +use crate::token::{self, DelimToken}; use crate::tokenstream::TokenStream; use syntax_pos::symbol::{kw, sym, Symbol}; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index c663995eb8f..1534d2723c8 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -17,7 +17,7 @@ use crate::source_map::{BytePos, Spanned}; use crate::parse::lexer::comments::doc_comment_style; use crate::parse; use crate::parse::PResult; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/feature_gate/check.rs b/src/libsyntax/feature_gate/check.rs index 5b1493ebc9b..ecff89ad59b 100644 --- a/src/libsyntax/feature_gate/check.rs +++ b/src/libsyntax/feature_gate/check.rs @@ -11,7 +11,7 @@ use crate::attr::{self, check_builtin_attribute}; use crate::source_map::Spanned; use crate::edition::{ALL_EDITIONS, Edition}; use crate::visit::{self, FnKind, Visitor}; -use crate::parse::token; +use crate::token; use crate::sess::ParseSess; use crate::symbol::{Symbol, sym}; use crate::tokenstream::TokenTree; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 7be6e6c7e18..4c134430f30 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -107,6 +107,7 @@ pub mod show_span; pub use syntax_pos::edition; pub use syntax_pos::symbol; pub mod sess; +pub mod token; pub mod tokenstream; pub mod visit; diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 7261601e144..0c90652526d 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -9,7 +9,7 @@ use crate::ast::*; use crate::source_map::{Spanned, respan}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::ptr::P; use crate::ThinVec; use crate::tokenstream::*; diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index a6dfed542d6..5499a3cae5f 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs index de301b1fc49..b9b82b2bcef 100644 --- a/src/libsyntax/parse/lexer/tests.rs +++ b/src/libsyntax/parse/lexer/tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; -use crate::parse::token; +use crate::token; use crate::with_default_globals; use errors::{Handler, emitter::EmitterWriter}; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index de8ac2c71e8..3c15a5e1dae 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -4,8 +4,8 @@ use syntax_pos::Span; use super::{StringReader, UnmatchedBrace}; use crate::print::pprust::token_to_string; -use crate::parse::token::{self, Token}; use crate::parse::PResult; +use crate::token::{self, Token}; use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { diff --git a/src/libsyntax/parse/lexer/unicode_chars.rs b/src/libsyntax/parse/lexer/unicode_chars.rs index 525b4215aff..6eb995b61d3 100644 --- a/src/libsyntax/parse/lexer/unicode_chars.rs +++ b/src/libsyntax/parse/lexer/unicode_chars.rs @@ -4,7 +4,7 @@ use super::StringReader; use errors::{Applicability, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span, symbol::kw}; -use crate::parse::token; +use crate::token; #[rustfmt::skip] // for line breaks const UNICODE_ARRAY: &[(char, &str, char)] = &[ diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index a8eeac59954..d4c9b7850c5 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -1,8 +1,8 @@ //! Code related to parsing literals. use crate::ast::{self, Lit, LitKind}; -use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; +use crate::token::{self, Token}; use crate::tokenstream::TokenTree; use log::debug; diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 5b020d3d288..7db0ca0b78d 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -2,7 +2,7 @@ use crate::ast; use crate::parse::parser::{Parser, emit_unclosed_delims, make_unclosed_delims_error}; -use crate::parse::token::Nonterminal; +use crate::token::{self, Nonterminal}; use crate::tokenstream::{self, TokenStream, TokenTree}; use crate::print::pprust; use crate::sess::ParseSess; @@ -25,7 +25,6 @@ mod tests; #[macro_use] pub mod parser; pub mod lexer; -pub mod token; crate mod classify; crate mod literal; diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 1c292661f24..aa0542a09a9 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -1,7 +1,7 @@ use super::{SeqSep, PResult, Parser, TokenType, PathStyle}; use crate::attr; use crate::ast; -use crate::parse::token::{self, Nonterminal, DelimToken}; +use crate::token::{self, Nonterminal, DelimToken}; use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; diff --git a/src/libsyntax/parse/parser/diagnostics.rs b/src/libsyntax/parse/parser/diagnostics.rs index 49a517a5c44..09f0b9b74b2 100644 --- a/src/libsyntax/parse/parser/diagnostics.rs +++ b/src/libsyntax/parse/parser/diagnostics.rs @@ -6,7 +6,7 @@ use crate::ast::{ self, Param, BinOpKind, BindingMode, BlockCheckMode, Expr, ExprKind, Ident, Item, ItemKind, Mutability, Pat, PatKind, PathSegment, QSelf, Ty, TyKind, }; -use crate::parse::token::{self, TokenKind, token_can_begin_expr}; +use crate::token::{self, TokenKind, token_can_begin_expr}; use crate::print::pprust; use crate::ptr::P; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/expr.rs b/src/libsyntax/parse/parser/expr.rs index 509e6482dcc..4b962201e85 100644 --- a/src/libsyntax/parse/parser/expr.rs +++ b/src/libsyntax/parse/parser/expr.rs @@ -12,7 +12,7 @@ use crate::ast::{ }; use crate::maybe_recover_from_interpolated_ty_qpath; use crate::parse::classify; -use crate::parse::token::{self, Token, TokenKind}; +use crate::token::{self, Token, TokenKind}; use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, Span}; diff --git a/src/libsyntax/parse/parser/generics.rs b/src/libsyntax/parse/parser/generics.rs index 3c094750b4d..cfc3768cac5 100644 --- a/src/libsyntax/parse/parser/generics.rs +++ b/src/libsyntax/parse/parser/generics.rs @@ -1,7 +1,7 @@ use super::{Parser, PResult}; use crate::ast::{self, WhereClause, GenericParam, GenericParamKind, GenericBounds, Attribute}; -use crate::parse::token; +use crate::token; use crate::source_map::DUMMY_SP; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/mod.rs b/src/libsyntax/parse/parser/mod.rs index 1284e89f195..6498b674a3b 100644 --- a/src/libsyntax/parse/parser/mod.rs +++ b/src/libsyntax/parse/parser/mod.rs @@ -18,7 +18,7 @@ use crate::ast::{ use crate::parse::{PResult, Directory, DirectoryOwnership}; use crate::parse::lexer::UnmatchedBrace; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; -use crate::parse::token::{self, Token, TokenKind, DelimToken}; +use crate::token::{self, Token, TokenKind, DelimToken}; use crate::print::pprust; use crate::ptr::P; use crate::sess::ParseSess; diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 242a17659a0..70e0dfc3e88 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -5,7 +5,7 @@ use super::diagnostics::Error; use crate::attr; use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate}; use crate::parse::{new_sub_parser_from_file, DirectoryOwnership}; -use crate::parse::token::{self, TokenKind}; +use crate::token::{self, TokenKind}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; use crate::symbol::sym; diff --git a/src/libsyntax/parse/parser/pat.rs b/src/libsyntax/parse/parser/pat.rs index fcb7c4d1323..c4a983188fa 100644 --- a/src/libsyntax/parse/parser/pat.rs +++ b/src/libsyntax/parse/parser/pat.rs @@ -5,7 +5,7 @@ use crate::ptr::P; use crate::ast::{self, Attribute, Pat, PatKind, FieldPat, RangeEnd, RangeSyntax, Mac}; use crate::ast::{BindingMode, Ident, Mutability, Path, QSelf, Expr, ExprKind}; use crate::mut_visit::{noop_visit_pat, noop_visit_mac, MutVisitor}; -use crate::parse::token; +use crate::token; use crate::print::pprust; use crate::source_map::{respan, Span, Spanned}; use crate::ThinVec; diff --git a/src/libsyntax/parse/parser/path.rs b/src/libsyntax/parse/parser/path.rs index 4438d61d9ee..2659e0c680e 100644 --- a/src/libsyntax/parse/parser/path.rs +++ b/src/libsyntax/parse/parser/path.rs @@ -3,7 +3,7 @@ use super::{Parser, PResult, TokenType}; use crate::{maybe_whole, ThinVec}; use crate::ast::{self, QSelf, Path, PathSegment, Ident, ParenthesizedArgs, AngleBracketedArgs}; use crate::ast::{AnonConst, GenericArg, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::{Span, BytePos}; use syntax_pos::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/stmt.rs b/src/libsyntax/parse/parser/stmt.rs index 12c530f3cbb..010fd9c37fd 100644 --- a/src/libsyntax/parse/parser/stmt.rs +++ b/src/libsyntax/parse/parser/stmt.rs @@ -9,7 +9,7 @@ use crate::{maybe_whole, ThinVec}; use crate::ast::{self, DUMMY_NODE_ID, Stmt, StmtKind, Local, Block, BlockCheckMode, Expr, ExprKind}; use crate::ast::{Attribute, AttrStyle, VisibilityKind, MacStmtStyle, Mac, MacDelimiter}; use crate::parse::{classify, DirectoryOwnership}; -use crate::parse::token; +use crate::token; use crate::source_map::{respan, Span}; use crate::symbol::{kw, sym}; diff --git a/src/libsyntax/parse/parser/ty.rs b/src/libsyntax/parse/parser/ty.rs index b770b90705c..4183416e628 100644 --- a/src/libsyntax/parse/parser/ty.rs +++ b/src/libsyntax/parse/parser/ty.rs @@ -6,7 +6,7 @@ use crate::ptr::P; use crate::ast::{self, Ty, TyKind, MutTy, BareFnTy, FunctionRetTy, GenericParam, Lifetime, Ident}; use crate::ast::{TraitBoundModifier, TraitObjectSyntax, GenericBound, GenericBounds, PolyTraitRef}; use crate::ast::{Mutability, AnonConst, Mac}; -use crate::parse::token::{self, Token}; +use crate::token::{self, Token}; use crate::source_map::Span; use crate::symbol::{kw}; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs index 201eaca947b..27ca2b6472f 100644 --- a/src/libsyntax/parse/tests.rs +++ b/src/libsyntax/parse/tests.rs @@ -4,7 +4,7 @@ use crate::ast::{self, Name, PatKind}; use crate::attr::first_attr_value_str_by_name; use crate::sess::ParseSess; use crate::parse::{PResult, new_parser_from_source_str}; -use crate::parse::token::Token; +use crate::token::Token; use crate::print::pprust::item_to_string; use crate::ptr::P; use crate::source_map::FilePathMapping; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs deleted file mode 100644 index 6f3da344ccf..00000000000 --- a/src/libsyntax/parse/token.rs +++ /dev/null @@ -1,728 +0,0 @@ -pub use BinOpToken::*; -pub use Nonterminal::*; -pub use DelimToken::*; -pub use LitKind::*; -pub use TokenKind::*; - -use crate::ast; -use crate::ptr::P; -use crate::symbol::kw; -use crate::tokenstream::TokenTree; - -use syntax_pos::symbol::Symbol; -use syntax_pos::{self, Span, DUMMY_SP}; - -use std::fmt; -use std::mem; -#[cfg(target_arch = "x86_64")] -use rustc_data_structures::static_assert_size; -use rustc_data_structures::sync::Lrc; - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum BinOpToken { - Plus, - Minus, - Star, - Slash, - Percent, - Caret, - And, - Or, - Shl, - Shr, -} - -/// A delimiter token. -#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] -pub enum DelimToken { - /// A round parenthesis (i.e., `(` or `)`). - Paren, - /// A square bracket (i.e., `[` or `]`). - Bracket, - /// A curly brace (i.e., `{` or `}`). - Brace, - /// An empty delimiter. - NoDelim, -} - -impl DelimToken { - pub fn len(self) -> usize { - if self == NoDelim { 0 } else { 1 } - } - - pub fn is_empty(self) -> bool { - self == NoDelim - } -} - -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum LitKind { - Bool, // AST only, must never appear in a `Token` - Byte, - Char, - Integer, - Float, - Str, - StrRaw(u16), // raw string delimited by `n` hash symbols - ByteStr, - ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols - Err, -} - -/// A literal token. -#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Lit { - pub kind: LitKind, - pub symbol: Symbol, - pub suffix: Option, -} - -impl fmt::Display for Lit { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Lit { kind, symbol, suffix } = *self; - match kind { - Byte => write!(f, "b'{}'", symbol)?, - Char => write!(f, "'{}'", symbol)?, - Str => write!(f, "\"{}\"", symbol)?, - StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - ByteStr => write!(f, "b\"{}\"", symbol)?, - ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", - delim="#".repeat(n as usize), - string=symbol)?, - Integer | - Float | - Bool | - Err => write!(f, "{}", symbol)?, - } - - if let Some(suffix) = suffix { - write!(f, "{}", suffix)?; - } - - Ok(()) - } -} - -impl LitKind { - /// An English article for the literal token kind. - crate fn article(self) -> &'static str { - match self { - Integer | Err => "an", - _ => "a", - } - } - - crate fn descr(self) -> &'static str { - match self { - Bool => panic!("literal token contains `Lit::Bool`"), - Byte => "byte", - Char => "char", - Integer => "integer", - Float => "float", - Str | StrRaw(..) => "string", - ByteStr | ByteStrRaw(..) => "byte string", - Err => "error", - } - } - - crate fn may_have_suffix(self) -> bool { - match self { - Integer | Float | Err => true, - _ => false, - } - } -} - -impl Lit { - pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { - Lit { kind, symbol, suffix } - } -} - -pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - token_can_begin_expr(&ident_token) -} - -pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - match ident_token.kind { - TokenKind::Ident(ident, _) => [ - kw::Async, - kw::Do, - kw::Box, - kw::Break, - kw::Continue, - kw::False, - kw::For, - kw::If, - kw::Let, - kw::Loop, - kw::Match, - kw::Move, - kw::Return, - kw::True, - kw::Unsafe, - kw::While, - kw::Yield, - kw::Static, - ].contains(&ident), - _=> false, - } -} - -fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { - let ident_token = Token::new(Ident(name, is_raw), span); - - !ident_token.is_reserved_ident() || - ident_token.is_path_segment_keyword() || - [ - kw::Underscore, - kw::For, - kw::Impl, - kw::Fn, - kw::Unsafe, - kw::Extern, - kw::Typeof, - kw::Dyn, - ].contains(&name) -} - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub enum TokenKind { - /* Expression-operator symbols. */ - Eq, - Lt, - Le, - EqEq, - Ne, - Ge, - Gt, - AndAnd, - OrOr, - Not, - Tilde, - BinOp(BinOpToken), - BinOpEq(BinOpToken), - - /* Structural symbols */ - At, - Dot, - DotDot, - DotDotDot, - DotDotEq, - Comma, - Semi, - Colon, - ModSep, - RArrow, - LArrow, - FatArrow, - Pound, - Dollar, - Question, - /// Used by proc macros for representing lifetimes, not generated by lexer right now. - SingleQuote, - /// An opening delimiter (e.g., `{`). - OpenDelim(DelimToken), - /// A closing delimiter (e.g., `}`). - CloseDelim(DelimToken), - - /* Literals */ - Literal(Lit), - - /* Name components */ - Ident(ast::Name, /* is_raw */ bool), - Lifetime(ast::Name), - - Interpolated(Lrc), - - // Can be expanded into several tokens. - /// A doc comment. - DocComment(ast::Name), - - // Junk. These carry no data because we don't really care about the data - // they *would* carry, and don't really want to allocate a new ident for - // them. Instead, users could extract that from the associated span. - - /// Whitespace. - Whitespace, - /// A comment. - Comment, - Shebang(ast::Name), - /// A completely invalid token which should be skipped. - Unknown(ast::Name), - - Eof, -} - -// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(target_arch = "x86_64")] -static_assert_size!(TokenKind, 16); - -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Token { - pub kind: TokenKind, - pub span: Span, -} - -impl TokenKind { - pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { - Literal(Lit::new(kind, symbol, suffix)) - } - - /// Returns tokens that are likely to be typed accidentally instead of the current token. - /// Enables better error recovery when the wrong token is found. - crate fn similar_tokens(&self) -> Option> { - match *self { - Comma => Some(vec![Dot, Lt, Semi]), - Semi => Some(vec![Colon, Comma]), - _ => None - } - } -} - -impl Token { - pub fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } - } - - /// Some token that will be thrown away later. - crate fn dummy() -> Self { - Token::new(TokenKind::Whitespace, DUMMY_SP) - } - - /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> Self { - Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) - } - - /// Return this token by value and leave a dummy token in its place. - pub fn take(&mut self) -> Self { - mem::replace(self, Token::dummy()) - } - - crate fn is_op(&self) -> bool { - match self.kind { - OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | - Ident(..) | Lifetime(..) | Interpolated(..) | - Whitespace | Comment | Shebang(..) | Eof => false, - _ => true, - } - } - - crate fn is_like_plus(&self) -> bool { - match self.kind { - BinOp(Plus) | BinOpEq(Plus) => true, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of an expression. - pub fn can_begin_expr(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_expr(name, self.span, is_raw), // value name or keyword - OpenDelim(..) | // tuple, array or block - Literal(..) | // literal - Not | // operator not - BinOp(Minus) | // unary minus - BinOp(Star) | // dereference - BinOp(Or) | OrOr | // closure - BinOp(And) | // reference - AndAnd | // double reference - // DotDotDot is no longer supported, but we need some way to display the error - DotDot | DotDotDot | DotDotEq | // range notation - Lt | BinOp(Shl) | // associated path - ModSep | // global path - Lifetime(..) | // labeled loop - Pound => true, // expression attributes - Interpolated(ref nt) => match **nt { - NtLiteral(..) | - NtIdent(..) | - NtExpr(..) | - NtBlock(..) | - NtPath(..) | - NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a type. - pub fn can_begin_type(&self) -> bool { - match self.kind { - Ident(name, is_raw) => - ident_can_begin_type(name, self.span, is_raw), // type name or keyword - OpenDelim(Paren) | // tuple - OpenDelim(Bracket) | // array - Not | // never - BinOp(Star) | // raw pointer - BinOp(And) | // reference - AndAnd | // double reference - Question | // maybe bound in trait object - Lifetime(..) | // lifetime bound in trait object - Lt | BinOp(Shl) | // associated path - ModSep => true, // global path - Interpolated(ref nt) => match **nt { - NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns `true` if the token can appear at the start of a const param. - crate fn can_begin_const_arg(&self) -> bool { - match self.kind { - OpenDelim(Brace) => true, - Interpolated(ref nt) => match **nt { - NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, - _ => false, - } - _ => self.can_begin_literal_or_bool(), - } - } - - /// Returns `true` if the token can appear at the start of a generic bound. - crate fn can_begin_bound(&self) -> bool { - self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || - self == &Question || self == &OpenDelim(Paren) - } - - /// Returns `true` if the token is any literal - pub fn is_lit(&self) -> bool { - match self.kind { - Literal(..) => true, - _ => false, - } - } - - /// Returns `true` if the token is any literal, a minus (which can prefix a literal, - /// for example a '-42', or one of the boolean idents). - pub fn can_begin_literal_or_bool(&self) -> bool { - match self.kind { - Literal(..) | BinOp(Minus) => true, - Ident(name, false) if name.is_bool_lit() => true, - Interpolated(ref nt) => match **nt { - NtLiteral(..) => true, - _ => false, - }, - _ => false, - } - } - - /// Returns an identifier if this token is an identifier. - pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { - match self.kind { - Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), - Interpolated(ref nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident, is_raw)), - _ => None, - }, - _ => None, - } - } - - /// Returns a lifetime identifier if this token is a lifetime. - pub fn lifetime(&self) -> Option { - match self.kind { - Lifetime(name) => Some(ast::Ident::new(name, self.span)), - Interpolated(ref nt) => match **nt { - NtLifetime(ident) => Some(ident), - _ => None, - }, - _ => None, - } - } - - /// Returns `true` if the token is an identifier. - pub fn is_ident(&self) -> bool { - self.ident().is_some() - } - - /// Returns `true` if the token is a lifetime. - crate fn is_lifetime(&self) -> bool { - self.lifetime().is_some() - } - - /// Returns `true` if the token is a identifier whose name is the given - /// string slice. - crate fn is_ident_named(&self, name: Symbol) -> bool { - self.ident().map_or(false, |(ident, _)| ident.name == name) - } - - /// Returns `true` if the token is an interpolated path. - fn is_path(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtPath(..) = **nt { - return true; - } - } - false - } - - /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? - /// That is, is this a pre-parsed expression dropped into the token stream - /// (which happens while parsing the result of macro expansion)? - crate fn is_whole_expr(&self) -> bool { - if let Interpolated(ref nt) = self.kind { - if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { - return true; - } - } - - false - } - - /// Returns `true` if the token is either the `mut` or `const` keyword. - crate fn is_mutability(&self) -> bool { - self.is_keyword(kw::Mut) || - self.is_keyword(kw::Const) - } - - crate fn is_qpath_start(&self) -> bool { - self == &Lt || self == &BinOp(Shl) - } - - crate fn is_path_start(&self) -> bool { - self == &ModSep || self.is_qpath_start() || self.is_path() || - self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() - } - - /// Returns `true` if the token is a given keyword, `kw`. - pub fn is_keyword(&self, kw: Symbol) -> bool { - self.is_non_raw_ident_where(|id| id.name == kw) - } - - crate fn is_path_segment_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) - } - - // Returns true for reserved identifiers used internally for elided lifetimes, - // unnamed method parameters, crate root module, error recovery etc. - crate fn is_special_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_special) - } - - /// Returns `true` if the token is a keyword used in the language. - crate fn is_used_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_used_keyword) - } - - /// Returns `true` if the token is a keyword reserved for possible future use. - crate fn is_unused_keyword(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) - } - - /// Returns `true` if the token is either a special identifier or a keyword. - pub fn is_reserved_ident(&self) -> bool { - self.is_non_raw_ident_where(ast::Ident::is_reserved) - } - - /// Returns `true` if the token is the identifier `true` or `false`. - crate fn is_bool_lit(&self) -> bool { - self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) - } - - /// Returns `true` if the token is a non-raw identifier for which `pred` holds. - fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { - match self.ident() { - Some((id, false)) => pred(id), - _ => false, - } - } - - crate fn glue(&self, joint: &Token) -> Option { - let kind = match self.kind { - Eq => match joint.kind { - Eq => EqEq, - Gt => FatArrow, - _ => return None, - }, - Lt => match joint.kind { - Eq => Le, - Lt => BinOp(Shl), - Le => BinOpEq(Shl), - BinOp(Minus) => LArrow, - _ => return None, - }, - Gt => match joint.kind { - Eq => Ge, - Gt => BinOp(Shr), - Ge => BinOpEq(Shr), - _ => return None, - }, - Not => match joint.kind { - Eq => Ne, - _ => return None, - }, - BinOp(op) => match joint.kind { - Eq => BinOpEq(op), - BinOp(And) if op == And => AndAnd, - BinOp(Or) if op == Or => OrOr, - Gt if op == Minus => RArrow, - _ => return None, - }, - Dot => match joint.kind { - Dot => DotDot, - DotDot => DotDotDot, - _ => return None, - }, - DotDot => match joint.kind { - Dot => DotDotDot, - Eq => DotDotEq, - _ => return None, - }, - Colon => match joint.kind { - Colon => ModSep, - _ => return None, - }, - SingleQuote => match joint.kind { - Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), - _ => return None, - }, - - Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | - DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | - Question | OpenDelim(..) | CloseDelim(..) | - Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, - }; - - Some(Token::new(kind, self.span.to(joint.span))) - } - - // See comments in `Nonterminal::to_tokenstream` for why we care about - // *probably* equal here rather than actual equality - crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { - if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { - return false - } - match (&self.kind, &other.kind) { - (&Eq, &Eq) | - (&Lt, &Lt) | - (&Le, &Le) | - (&EqEq, &EqEq) | - (&Ne, &Ne) | - (&Ge, &Ge) | - (&Gt, &Gt) | - (&AndAnd, &AndAnd) | - (&OrOr, &OrOr) | - (&Not, &Not) | - (&Tilde, &Tilde) | - (&At, &At) | - (&Dot, &Dot) | - (&DotDot, &DotDot) | - (&DotDotDot, &DotDotDot) | - (&DotDotEq, &DotDotEq) | - (&Comma, &Comma) | - (&Semi, &Semi) | - (&Colon, &Colon) | - (&ModSep, &ModSep) | - (&RArrow, &RArrow) | - (&LArrow, &LArrow) | - (&FatArrow, &FatArrow) | - (&Pound, &Pound) | - (&Dollar, &Dollar) | - (&Question, &Question) | - (&Whitespace, &Whitespace) | - (&Comment, &Comment) | - (&Eof, &Eof) => true, - - (&BinOp(a), &BinOp(b)) | - (&BinOpEq(a), &BinOpEq(b)) => a == b, - - (&OpenDelim(a), &OpenDelim(b)) | - (&CloseDelim(a), &CloseDelim(b)) => a == b, - - (&DocComment(a), &DocComment(b)) | - (&Shebang(a), &Shebang(b)) => a == b, - - (&Literal(a), &Literal(b)) => a == b, - - (&Lifetime(a), &Lifetime(b)) => a == b, - (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || - a == kw::DollarCrate || - c == kw::DollarCrate), - - (&Interpolated(_), &Interpolated(_)) => false, - - _ => panic!("forgot to add a token?"), - } - } -} - -impl PartialEq for Token { - fn eq(&self, rhs: &TokenKind) -> bool { - self.kind == *rhs - } -} - -#[derive(Clone, RustcEncodable, RustcDecodable)] -/// For interpolation during macro expansion. -pub enum Nonterminal { - NtItem(P), - NtBlock(P), - NtStmt(ast::Stmt), - NtPat(P), - NtExpr(P), - NtTy(P), - NtIdent(ast::Ident, /* is_raw */ bool), - NtLifetime(ast::Ident), - NtLiteral(P), - /// Stuff inside brackets for attributes - NtMeta(ast::AttrItem), - NtPath(ast::Path), - NtVis(ast::Visibility), - NtTT(TokenTree), - // Used only for passing items to proc macro attributes (they are not - // strictly necessary for that, `Annotatable` can be converted into - // tokens directly, but doing that naively regresses pretty-printing). - NtTraitItem(ast::TraitItem), - NtImplItem(ast::ImplItem), - NtForeignItem(ast::ForeignItem), -} - -impl PartialEq for Nonterminal { - fn eq(&self, rhs: &Self) -> bool { - match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => - ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, - // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them - // correctly based on data from AST. This will prevent them from matching each other - // in macros. The comparison will become possible only when each nonterminal has an - // attached token stream from which it was parsed. - _ => false, - } - } -} - -impl fmt::Debug for Nonterminal { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - NtItem(..) => f.pad("NtItem(..)"), - NtBlock(..) => f.pad("NtBlock(..)"), - NtStmt(..) => f.pad("NtStmt(..)"), - NtPat(..) => f.pad("NtPat(..)"), - NtExpr(..) => f.pad("NtExpr(..)"), - NtTy(..) => f.pad("NtTy(..)"), - NtIdent(..) => f.pad("NtIdent(..)"), - NtLiteral(..) => f.pad("NtLiteral(..)"), - NtMeta(..) => f.pad("NtMeta(..)"), - NtPath(..) => f.pad("NtPath(..)"), - NtTT(..) => f.pad("NtTT(..)"), - NtImplItem(..) => f.pad("NtImplItem(..)"), - NtTraitItem(..) => f.pad("NtTraitItem(..)"), - NtForeignItem(..) => f.pad("NtForeignItem(..)"), - NtVis(..) => f.pad("NtVis(..)"), - NtLifetime(..) => f.pad("NtLifetime(..)"), - } - } -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 1d59c13a9d0..049f34efc26 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -4,7 +4,7 @@ use crate::ast::{Attribute, MacDelimiter, GenericArg}; use crate::util::parser::{self, AssocOp, Fixity}; use crate::attr; use crate::source_map::{self, SourceMap, Spanned}; -use crate::parse::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; +use crate::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; use crate::parse::lexer::comments; use crate::parse; use crate::print::pp::{self, Breaks}; diff --git a/src/libsyntax/token.rs b/src/libsyntax/token.rs new file mode 100644 index 00000000000..6f3da344ccf --- /dev/null +++ b/src/libsyntax/token.rs @@ -0,0 +1,728 @@ +pub use BinOpToken::*; +pub use Nonterminal::*; +pub use DelimToken::*; +pub use LitKind::*; +pub use TokenKind::*; + +use crate::ast; +use crate::ptr::P; +use crate::symbol::kw; +use crate::tokenstream::TokenTree; + +use syntax_pos::symbol::Symbol; +use syntax_pos::{self, Span, DUMMY_SP}; + +use std::fmt; +use std::mem; +#[cfg(target_arch = "x86_64")] +use rustc_data_structures::static_assert_size; +use rustc_data_structures::sync::Lrc; + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum BinOpToken { + Plus, + Minus, + Star, + Slash, + Percent, + Caret, + And, + Or, + Shl, + Shr, +} + +/// A delimiter token. +#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] +pub enum DelimToken { + /// A round parenthesis (i.e., `(` or `)`). + Paren, + /// A square bracket (i.e., `[` or `]`). + Bracket, + /// A curly brace (i.e., `{` or `}`). + Brace, + /// An empty delimiter. + NoDelim, +} + +impl DelimToken { + pub fn len(self) -> usize { + if self == NoDelim { 0 } else { 1 } + } + + pub fn is_empty(self) -> bool { + self == NoDelim + } +} + +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum LitKind { + Bool, // AST only, must never appear in a `Token` + Byte, + Char, + Integer, + Float, + Str, + StrRaw(u16), // raw string delimited by `n` hash symbols + ByteStr, + ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols + Err, +} + +/// A literal token. +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Lit { + pub kind: LitKind, + pub symbol: Symbol, + pub suffix: Option, +} + +impl fmt::Display for Lit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Lit { kind, symbol, suffix } = *self; + match kind { + Byte => write!(f, "b'{}'", symbol)?, + Char => write!(f, "'{}'", symbol)?, + Str => write!(f, "\"{}\"", symbol)?, + StrRaw(n) => write!(f, "r{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + ByteStr => write!(f, "b\"{}\"", symbol)?, + ByteStrRaw(n) => write!(f, "br{delim}\"{string}\"{delim}", + delim="#".repeat(n as usize), + string=symbol)?, + Integer | + Float | + Bool | + Err => write!(f, "{}", symbol)?, + } + + if let Some(suffix) = suffix { + write!(f, "{}", suffix)?; + } + + Ok(()) + } +} + +impl LitKind { + /// An English article for the literal token kind. + crate fn article(self) -> &'static str { + match self { + Integer | Err => "an", + _ => "a", + } + } + + crate fn descr(self) -> &'static str { + match self { + Bool => panic!("literal token contains `Lit::Bool`"), + Byte => "byte", + Char => "char", + Integer => "integer", + Float => "float", + Str | StrRaw(..) => "string", + ByteStr | ByteStrRaw(..) => "byte string", + Err => "error", + } + } + + crate fn may_have_suffix(self) -> bool { + match self { + Integer | Float | Err => true, + _ => false, + } + } +} + +impl Lit { + pub fn new(kind: LitKind, symbol: Symbol, suffix: Option) -> Lit { + Lit { kind, symbol, suffix } + } +} + +pub(crate) fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + token_can_begin_expr(&ident_token) +} + +pub(crate) fn token_can_begin_expr(ident_token: &Token) -> bool { + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + match ident_token.kind { + TokenKind::Ident(ident, _) => [ + kw::Async, + kw::Do, + kw::Box, + kw::Break, + kw::Continue, + kw::False, + kw::For, + kw::If, + kw::Let, + kw::Loop, + kw::Match, + kw::Move, + kw::Return, + kw::True, + kw::Unsafe, + kw::While, + kw::Yield, + kw::Static, + ].contains(&ident), + _=> false, + } +} + +fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { + let ident_token = Token::new(Ident(name, is_raw), span); + + !ident_token.is_reserved_ident() || + ident_token.is_path_segment_keyword() || + [ + kw::Underscore, + kw::For, + kw::Impl, + kw::Fn, + kw::Unsafe, + kw::Extern, + kw::Typeof, + kw::Dyn, + ].contains(&name) +} + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum TokenKind { + /* Expression-operator symbols. */ + Eq, + Lt, + Le, + EqEq, + Ne, + Ge, + Gt, + AndAnd, + OrOr, + Not, + Tilde, + BinOp(BinOpToken), + BinOpEq(BinOpToken), + + /* Structural symbols */ + At, + Dot, + DotDot, + DotDotDot, + DotDotEq, + Comma, + Semi, + Colon, + ModSep, + RArrow, + LArrow, + FatArrow, + Pound, + Dollar, + Question, + /// Used by proc macros for representing lifetimes, not generated by lexer right now. + SingleQuote, + /// An opening delimiter (e.g., `{`). + OpenDelim(DelimToken), + /// A closing delimiter (e.g., `}`). + CloseDelim(DelimToken), + + /* Literals */ + Literal(Lit), + + /* Name components */ + Ident(ast::Name, /* is_raw */ bool), + Lifetime(ast::Name), + + Interpolated(Lrc), + + // Can be expanded into several tokens. + /// A doc comment. + DocComment(ast::Name), + + // Junk. These carry no data because we don't really care about the data + // they *would* carry, and don't really want to allocate a new ident for + // them. Instead, users could extract that from the associated span. + + /// Whitespace. + Whitespace, + /// A comment. + Comment, + Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), + + Eof, +} + +// `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert_size!(TokenKind, 16); + +#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub struct Token { + pub kind: TokenKind, + pub span: Span, +} + +impl TokenKind { + pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { + Literal(Lit::new(kind, symbol, suffix)) + } + + /// Returns tokens that are likely to be typed accidentally instead of the current token. + /// Enables better error recovery when the wrong token is found. + crate fn similar_tokens(&self) -> Option> { + match *self { + Comma => Some(vec![Dot, Lt, Semi]), + Semi => Some(vec![Colon, Comma]), + _ => None + } + } +} + +impl Token { + pub fn new(kind: TokenKind, span: Span) -> Self { + Token { kind, span } + } + + /// Some token that will be thrown away later. + crate fn dummy() -> Self { + Token::new(TokenKind::Whitespace, DUMMY_SP) + } + + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. + pub fn from_ast_ident(ident: ast::Ident) -> Self { + Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) + } + + /// Return this token by value and leave a dummy token in its place. + pub fn take(&mut self) -> Self { + mem::replace(self, Token::dummy()) + } + + crate fn is_op(&self) -> bool { + match self.kind { + OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | + Ident(..) | Lifetime(..) | Interpolated(..) | + Whitespace | Comment | Shebang(..) | Eof => false, + _ => true, + } + } + + crate fn is_like_plus(&self) -> bool { + match self.kind { + BinOp(Plus) | BinOpEq(Plus) => true, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of an expression. + pub fn can_begin_expr(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_expr(name, self.span, is_raw), // value name or keyword + OpenDelim(..) | // tuple, array or block + Literal(..) | // literal + Not | // operator not + BinOp(Minus) | // unary minus + BinOp(Star) | // dereference + BinOp(Or) | OrOr | // closure + BinOp(And) | // reference + AndAnd | // double reference + // DotDotDot is no longer supported, but we need some way to display the error + DotDot | DotDotDot | DotDotEq | // range notation + Lt | BinOp(Shl) | // associated path + ModSep | // global path + Lifetime(..) | // labeled loop + Pound => true, // expression attributes + Interpolated(ref nt) => match **nt { + NtLiteral(..) | + NtIdent(..) | + NtExpr(..) | + NtBlock(..) | + NtPath(..) | + NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a type. + pub fn can_begin_type(&self) -> bool { + match self.kind { + Ident(name, is_raw) => + ident_can_begin_type(name, self.span, is_raw), // type name or keyword + OpenDelim(Paren) | // tuple + OpenDelim(Bracket) | // array + Not | // never + BinOp(Star) | // raw pointer + BinOp(And) | // reference + AndAnd | // double reference + Question | // maybe bound in trait object + Lifetime(..) | // lifetime bound in trait object + Lt | BinOp(Shl) | // associated path + ModSep => true, // global path + Interpolated(ref nt) => match **nt { + NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns `true` if the token can appear at the start of a const param. + crate fn can_begin_const_arg(&self) -> bool { + match self.kind { + OpenDelim(Brace) => true, + Interpolated(ref nt) => match **nt { + NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, + _ => false, + } + _ => self.can_begin_literal_or_bool(), + } + } + + /// Returns `true` if the token can appear at the start of a generic bound. + crate fn can_begin_bound(&self) -> bool { + self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || + self == &Question || self == &OpenDelim(Paren) + } + + /// Returns `true` if the token is any literal + pub fn is_lit(&self) -> bool { + match self.kind { + Literal(..) => true, + _ => false, + } + } + + /// Returns `true` if the token is any literal, a minus (which can prefix a literal, + /// for example a '-42', or one of the boolean idents). + pub fn can_begin_literal_or_bool(&self) -> bool { + match self.kind { + Literal(..) | BinOp(Minus) => true, + Ident(name, false) if name.is_bool_lit() => true, + Interpolated(ref nt) => match **nt { + NtLiteral(..) => true, + _ => false, + }, + _ => false, + } + } + + /// Returns an identifier if this token is an identifier. + pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { + match self.kind { + Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), + Interpolated(ref nt) => match **nt { + NtIdent(ident, is_raw) => Some((ident, is_raw)), + _ => None, + }, + _ => None, + } + } + + /// Returns a lifetime identifier if this token is a lifetime. + pub fn lifetime(&self) -> Option { + match self.kind { + Lifetime(name) => Some(ast::Ident::new(name, self.span)), + Interpolated(ref nt) => match **nt { + NtLifetime(ident) => Some(ident), + _ => None, + }, + _ => None, + } + } + + /// Returns `true` if the token is an identifier. + pub fn is_ident(&self) -> bool { + self.ident().is_some() + } + + /// Returns `true` if the token is a lifetime. + crate fn is_lifetime(&self) -> bool { + self.lifetime().is_some() + } + + /// Returns `true` if the token is a identifier whose name is the given + /// string slice. + crate fn is_ident_named(&self, name: Symbol) -> bool { + self.ident().map_or(false, |(ident, _)| ident.name == name) + } + + /// Returns `true` if the token is an interpolated path. + fn is_path(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtPath(..) = **nt { + return true; + } + } + false + } + + /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? + /// That is, is this a pre-parsed expression dropped into the token stream + /// (which happens while parsing the result of macro expansion)? + crate fn is_whole_expr(&self) -> bool { + if let Interpolated(ref nt) = self.kind { + if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { + return true; + } + } + + false + } + + /// Returns `true` if the token is either the `mut` or `const` keyword. + crate fn is_mutability(&self) -> bool { + self.is_keyword(kw::Mut) || + self.is_keyword(kw::Const) + } + + crate fn is_qpath_start(&self) -> bool { + self == &Lt || self == &BinOp(Shl) + } + + crate fn is_path_start(&self) -> bool { + self == &ModSep || self.is_qpath_start() || self.is_path() || + self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() + } + + /// Returns `true` if the token is a given keyword, `kw`. + pub fn is_keyword(&self, kw: Symbol) -> bool { + self.is_non_raw_ident_where(|id| id.name == kw) + } + + crate fn is_path_segment_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) + } + + // Returns true for reserved identifiers used internally for elided lifetimes, + // unnamed method parameters, crate root module, error recovery etc. + crate fn is_special_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_special) + } + + /// Returns `true` if the token is a keyword used in the language. + crate fn is_used_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_used_keyword) + } + + /// Returns `true` if the token is a keyword reserved for possible future use. + crate fn is_unused_keyword(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) + } + + /// Returns `true` if the token is either a special identifier or a keyword. + pub fn is_reserved_ident(&self) -> bool { + self.is_non_raw_ident_where(ast::Ident::is_reserved) + } + + /// Returns `true` if the token is the identifier `true` or `false`. + crate fn is_bool_lit(&self) -> bool { + self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) + } + + /// Returns `true` if the token is a non-raw identifier for which `pred` holds. + fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { + match self.ident() { + Some((id, false)) => pred(id), + _ => false, + } + } + + crate fn glue(&self, joint: &Token) -> Option { + let kind = match self.kind { + Eq => match joint.kind { + Eq => EqEq, + Gt => FatArrow, + _ => return None, + }, + Lt => match joint.kind { + Eq => Le, + Lt => BinOp(Shl), + Le => BinOpEq(Shl), + BinOp(Minus) => LArrow, + _ => return None, + }, + Gt => match joint.kind { + Eq => Ge, + Gt => BinOp(Shr), + Ge => BinOpEq(Shr), + _ => return None, + }, + Not => match joint.kind { + Eq => Ne, + _ => return None, + }, + BinOp(op) => match joint.kind { + Eq => BinOpEq(op), + BinOp(And) if op == And => AndAnd, + BinOp(Or) if op == Or => OrOr, + Gt if op == Minus => RArrow, + _ => return None, + }, + Dot => match joint.kind { + Dot => DotDot, + DotDot => DotDotDot, + _ => return None, + }, + DotDot => match joint.kind { + Dot => DotDotDot, + Eq => DotDotEq, + _ => return None, + }, + Colon => match joint.kind { + Colon => ModSep, + _ => return None, + }, + SingleQuote => match joint.kind { + Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), + _ => return None, + }, + + Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | + DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | + Question | OpenDelim(..) | CloseDelim(..) | + Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, + }; + + Some(Token::new(kind, self.span.to(joint.span))) + } + + // See comments in `Nonterminal::to_tokenstream` for why we care about + // *probably* equal here rather than actual equality + crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { + if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { + return false + } + match (&self.kind, &other.kind) { + (&Eq, &Eq) | + (&Lt, &Lt) | + (&Le, &Le) | + (&EqEq, &EqEq) | + (&Ne, &Ne) | + (&Ge, &Ge) | + (&Gt, &Gt) | + (&AndAnd, &AndAnd) | + (&OrOr, &OrOr) | + (&Not, &Not) | + (&Tilde, &Tilde) | + (&At, &At) | + (&Dot, &Dot) | + (&DotDot, &DotDot) | + (&DotDotDot, &DotDotDot) | + (&DotDotEq, &DotDotEq) | + (&Comma, &Comma) | + (&Semi, &Semi) | + (&Colon, &Colon) | + (&ModSep, &ModSep) | + (&RArrow, &RArrow) | + (&LArrow, &LArrow) | + (&FatArrow, &FatArrow) | + (&Pound, &Pound) | + (&Dollar, &Dollar) | + (&Question, &Question) | + (&Whitespace, &Whitespace) | + (&Comment, &Comment) | + (&Eof, &Eof) => true, + + (&BinOp(a), &BinOp(b)) | + (&BinOpEq(a), &BinOpEq(b)) => a == b, + + (&OpenDelim(a), &OpenDelim(b)) | + (&CloseDelim(a), &CloseDelim(b)) => a == b, + + (&DocComment(a), &DocComment(b)) | + (&Shebang(a), &Shebang(b)) => a == b, + + (&Literal(a), &Literal(b)) => a == b, + + (&Lifetime(a), &Lifetime(b)) => a == b, + (&Ident(a, b), &Ident(c, d)) => b == d && (a == c || + a == kw::DollarCrate || + c == kw::DollarCrate), + + (&Interpolated(_), &Interpolated(_)) => false, + + _ => panic!("forgot to add a token?"), + } + } +} + +impl PartialEq for Token { + fn eq(&self, rhs: &TokenKind) -> bool { + self.kind == *rhs + } +} + +#[derive(Clone, RustcEncodable, RustcDecodable)] +/// For interpolation during macro expansion. +pub enum Nonterminal { + NtItem(P), + NtBlock(P), + NtStmt(ast::Stmt), + NtPat(P), + NtExpr(P), + NtTy(P), + NtIdent(ast::Ident, /* is_raw */ bool), + NtLifetime(ast::Ident), + NtLiteral(P), + /// Stuff inside brackets for attributes + NtMeta(ast::AttrItem), + NtPath(ast::Path), + NtVis(ast::Visibility), + NtTT(TokenTree), + // Used only for passing items to proc macro attributes (they are not + // strictly necessary for that, `Annotatable` can be converted into + // tokens directly, but doing that naively regresses pretty-printing). + NtTraitItem(ast::TraitItem), + NtImplItem(ast::ImplItem), + NtForeignItem(ast::ForeignItem), +} + +impl PartialEq for Nonterminal { + fn eq(&self, rhs: &Self) -> bool { + match (self, rhs) { + (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => + ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs, + (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, + (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, + // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them + // correctly based on data from AST. This will prevent them from matching each other + // in macros. The comparison will become possible only when each nonterminal has an + // attached token stream from which it was parsed. + _ => false, + } + } +} + +impl fmt::Debug for Nonterminal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + NtItem(..) => f.pad("NtItem(..)"), + NtBlock(..) => f.pad("NtBlock(..)"), + NtStmt(..) => f.pad("NtStmt(..)"), + NtPat(..) => f.pad("NtPat(..)"), + NtExpr(..) => f.pad("NtExpr(..)"), + NtTy(..) => f.pad("NtTy(..)"), + NtIdent(..) => f.pad("NtIdent(..)"), + NtLiteral(..) => f.pad("NtLiteral(..)"), + NtMeta(..) => f.pad("NtMeta(..)"), + NtPath(..) => f.pad("NtPath(..)"), + NtTT(..) => f.pad("NtTT(..)"), + NtImplItem(..) => f.pad("NtImplItem(..)"), + NtTraitItem(..) => f.pad("NtTraitItem(..)"), + NtForeignItem(..) => f.pad("NtForeignItem(..)"), + NtVis(..) => f.pad("NtVis(..)"), + NtLifetime(..) => f.pad("NtLifetime(..)"), + } + } +} diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index a51d208704a..6e1bb85ce1a 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -13,7 +13,7 @@ //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking //! ownership of the original. -use crate::parse::token::{self, DelimToken, Token, TokenKind}; +use crate::token::{self, DelimToken, Token, TokenKind}; use syntax_pos::{Span, DUMMY_SP}; #[cfg(target_arch = "x86_64")] diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 982755e8680..edb708d7e97 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, Token, BinOpToken}; +use crate::token::{self, Token, BinOpToken}; use crate::symbol::kw; use crate::ast::{self, BinOpKind}; diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 117787d08c7..cfd160fd577 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -14,7 +14,7 @@ //! those that are created by the expansion of a macro. use crate::ast::*; -use crate::parse::token::Token; +use crate::token::Token; use crate::tokenstream::{TokenTree, TokenStream}; use syntax_pos::Span; diff --git a/src/libsyntax_expand/base.rs b/src/libsyntax_expand/base.rs index d02251eb746..47835c92659 100644 --- a/src/libsyntax_expand/base.rs +++ b/src/libsyntax_expand/base.rs @@ -6,11 +6,11 @@ use syntax::source_map::SourceMap; use syntax::edition::Edition; use syntax::mut_visit::{self, MutVisitor}; use syntax::parse::{self, parser, DirectoryOwnership}; -use syntax::parse::token; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Ident, Symbol}; use syntax::{ThinVec, MACRO_ARGUMENTS}; +use syntax::token; use syntax::tokenstream::{self, TokenStream}; use syntax::visit::Visitor; diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs index 7dbc7787010..6d1f0abe49e 100644 --- a/src/libsyntax_expand/expand.rs +++ b/src/libsyntax_expand/expand.rs @@ -13,12 +13,12 @@ use syntax::config::StripUnconfigured; use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; use syntax::mut_visit::*; use syntax::parse::{DirectoryOwnership, PResult}; -use syntax::parse::token; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; use syntax::sess::ParseSess; use syntax::symbol::{sym, Symbol}; +use syntax::token; use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::visit::{self, Visitor}; use syntax::util::map_in_place::MapInPlace; diff --git a/src/libsyntax_expand/mbe.rs b/src/libsyntax_expand/mbe.rs index 06e0cde3ad8..6964d01b719 100644 --- a/src/libsyntax_expand/mbe.rs +++ b/src/libsyntax_expand/mbe.rs @@ -10,7 +10,7 @@ crate mod macro_rules; crate mod quoted; use syntax::ast; -use syntax::parse::token::{self, Token, TokenKind}; +use syntax::token::{self, Token, TokenKind}; use syntax::tokenstream::{DelimSpan}; use syntax_pos::Span; diff --git a/src/libsyntax_expand/mbe/macro_check.rs b/src/libsyntax_expand/mbe/macro_check.rs index 50abda8d45e..25754ed4217 100644 --- a/src/libsyntax_expand/mbe/macro_check.rs +++ b/src/libsyntax_expand/mbe/macro_check.rs @@ -108,7 +108,7 @@ use crate::mbe::{KleeneToken, TokenTree}; use syntax::ast::NodeId; use syntax::early_buffered_lints::BufferedEarlyLintId; -use syntax::parse::token::{DelimToken, Token, TokenKind}; +use syntax::token::{DelimToken, Token, TokenKind}; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; diff --git a/src/libsyntax_expand/mbe/macro_parser.rs b/src/libsyntax_expand/mbe/macro_parser.rs index 3efe22626a9..950e7c0f286 100644 --- a/src/libsyntax_expand/mbe/macro_parser.rs +++ b/src/libsyntax_expand/mbe/macro_parser.rs @@ -79,10 +79,10 @@ use crate::mbe::{self, TokenTree}; use syntax::ast::{Ident, Name}; use syntax::parse::{Directory, PResult}; use syntax::parse::parser::{Parser, PathStyle}; -use syntax::parse::token::{self, DocComment, Nonterminal, Token}; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, DocComment, Nonterminal, Token}; use syntax::tokenstream::{DelimSpan, TokenStream}; use errors::FatalError; diff --git a/src/libsyntax_expand/mbe/macro_rules.rs b/src/libsyntax_expand/mbe/macro_rules.rs index 55719907403..a5fc301fbf6 100644 --- a/src/libsyntax_expand/mbe/macro_rules.rs +++ b/src/libsyntax_expand/mbe/macro_rules.rs @@ -13,12 +13,11 @@ use syntax::attr::{self, TransparencyError}; use syntax::edition::Edition; use syntax::feature_gate::Features; use syntax::parse::parser::Parser; -use syntax::parse::token::TokenKind::*; -use syntax::parse::token::{self, NtTT, Token}; use syntax::parse::Directory; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym, Symbol}; +use syntax::token::{self, NtTT, Token, TokenKind::*}; use syntax::tokenstream::{DelimSpan, TokenStream}; use errors::{DiagnosticBuilder, FatalError}; diff --git a/src/libsyntax_expand/mbe/quoted.rs b/src/libsyntax_expand/mbe/quoted.rs index cedd59233ad..dec504c0d97 100644 --- a/src/libsyntax_expand/mbe/quoted.rs +++ b/src/libsyntax_expand/mbe/quoted.rs @@ -2,10 +2,10 @@ use crate::mbe::macro_parser; use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; use syntax::ast; -use syntax::parse::token::{self, Token}; use syntax::print::pprust; use syntax::sess::ParseSess; use syntax::symbol::kw; +use syntax::token::{self, Token}; use syntax::tokenstream; use syntax_pos::Span; diff --git a/src/libsyntax_expand/mbe/transcribe.rs b/src/libsyntax_expand/mbe/transcribe.rs index 6f060103ef4..4092d4b97de 100644 --- a/src/libsyntax_expand/mbe/transcribe.rs +++ b/src/libsyntax_expand/mbe/transcribe.rs @@ -4,7 +4,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use syntax::ast::{Ident, Mac}; use syntax::mut_visit::{self, MutVisitor}; -use syntax::parse::token::{self, NtTT, Token}; +use syntax::token::{self, NtTT, Token}; use syntax::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use smallvec::{smallvec, SmallVec}; diff --git a/src/libsyntax_expand/proc_macro.rs b/src/libsyntax_expand/proc_macro.rs index 1f4c481e3ea..51c368bbaa6 100644 --- a/src/libsyntax_expand/proc_macro.rs +++ b/src/libsyntax_expand/proc_macro.rs @@ -4,8 +4,9 @@ use crate::proc_macro_server; use syntax::ast::{self, ItemKind, Attribute, Mac}; use syntax::attr::{mark_used, mark_known}; use syntax::errors::{Applicability, FatalError}; -use syntax::parse::{self, token}; +use syntax::parse; use syntax::symbol::sym; +use syntax::token; use syntax::tokenstream::{self, TokenStream}; use syntax::visit::Visitor; diff --git a/src/libsyntax_expand/proc_macro_server.rs b/src/libsyntax_expand/proc_macro_server.rs index 4ce99cfe73b..67a15e48205 100644 --- a/src/libsyntax_expand/proc_macro_server.rs +++ b/src/libsyntax_expand/proc_macro_server.rs @@ -1,10 +1,11 @@ use crate::base::ExtCtxt; use syntax::ast; -use syntax::parse::{self, token}; +use syntax::parse::{self}; use syntax::parse::lexer::comments; use syntax::print::pprust; use syntax::sess::ParseSess; +use syntax::token; use syntax::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; use errors::Diagnostic; @@ -52,7 +53,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> { fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) -> Self { - use syntax::parse::token::*; + use syntax::token::*; let joint = is_joint == Joint; let Token { kind, span } = match tree { @@ -193,7 +194,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> impl ToInternal for TokenTree { fn to_internal(self) -> TokenStream { - use syntax::parse::token::*; + use syntax::token::*; let (ch, joint, span) = match self { TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), diff --git a/src/libsyntax_ext/asm.rs b/src/libsyntax_ext/asm.rs index 8c9a34713ea..539d777105d 100644 --- a/src/libsyntax_ext/asm.rs +++ b/src/libsyntax_ext/asm.rs @@ -8,7 +8,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::ptr::P; use syntax::symbol::{kw, sym, Symbol}; use syntax::ast::AsmDialect; diff --git a/src/libsyntax_ext/assert.rs b/src/libsyntax_ext/assert.rs index f4d1f7fb09c..a15423b7ad8 100644 --- a/src/libsyntax_ext/assert.rs +++ b/src/libsyntax_ext/assert.rs @@ -2,7 +2,7 @@ use errors::{Applicability, DiagnosticBuilder}; use syntax::ast::{self, *}; use syntax_expand::base::*; -use syntax::parse::token::{self, TokenKind}; +use syntax::token::{self, TokenKind}; use syntax::parse::parser::Parser; use syntax::print::pprust; use syntax::ptr::P; diff --git a/src/libsyntax_ext/cfg.rs b/src/libsyntax_ext/cfg.rs index 9e693f29c5a..583236d9754 100644 --- a/src/libsyntax_ext/cfg.rs +++ b/src/libsyntax_ext/cfg.rs @@ -8,7 +8,7 @@ use syntax::ast; use syntax_expand::base::{self, *}; use syntax::attr; use syntax::tokenstream::TokenStream; -use syntax::parse::token; +use syntax::token; use syntax_pos::Span; pub fn expand_cfg( diff --git a/src/libsyntax_ext/cmdline_attrs.rs b/src/libsyntax_ext/cmdline_attrs.rs index 2d981526a39..171f2405573 100644 --- a/src/libsyntax_ext/cmdline_attrs.rs +++ b/src/libsyntax_ext/cmdline_attrs.rs @@ -2,7 +2,8 @@ use syntax::ast::{self, AttrItem, AttrStyle}; use syntax::attr::mk_attr; -use syntax::parse::{self, token}; +use syntax::parse; +use syntax::token; use syntax::sess::ParseSess; use syntax_expand::panictry; use syntax_pos::FileName; diff --git a/src/libsyntax_ext/concat_idents.rs b/src/libsyntax_ext/concat_idents.rs index a132a4136ea..8a1bc56cf1c 100644 --- a/src/libsyntax_ext/concat_idents.rs +++ b/src/libsyntax_ext/concat_idents.rs @@ -2,7 +2,7 @@ use rustc_data_structures::thin_vec::ThinVec; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::ptr::P; use syntax_pos::Span; use syntax_pos::symbol::Symbol; diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs index 314c2eefd4c..e25ba7b1783 100644 --- a/src/libsyntax_ext/format.rs +++ b/src/libsyntax_ext/format.rs @@ -9,7 +9,7 @@ use errors::pluralize; use syntax::ast; use syntax_expand::base::{self, *}; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax::symbol::{Symbol, sym}; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/global_asm.rs b/src/libsyntax_ext/global_asm.rs index 879ae1e4215..8a8ce9a7f14 100644 --- a/src/libsyntax_ext/global_asm.rs +++ b/src/libsyntax_ext/global_asm.rs @@ -13,7 +13,7 @@ use errors::DiagnosticBuilder; use syntax::ast; use syntax::source_map::respan; use syntax_expand::base::{self, *}; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax_pos::Span; use syntax::tokenstream::TokenStream; diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index 1ca9422eb9d..cee1b97af55 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -4,12 +4,12 @@ use syntax::ast::*; use syntax::attr; use syntax::edition::Edition; -use syntax_expand::base::{Resolver, NamedSyntaxExtension}; -use syntax::parse::token; use syntax::ptr::P; use syntax::source_map::respan; use syntax::symbol::sym; +use syntax::token; use syntax::tokenstream::*; +use syntax_expand::base::{Resolver, NamedSyntaxExtension}; use syntax_pos::{Span, DUMMY_SP}; use syntax_pos::hygiene::{ExpnData, ExpnKind, AstPass}; diff --git a/src/libsyntax_ext/source_util.rs b/src/libsyntax_ext/source_util.rs index f6c58fcdfa1..7e47b40714d 100644 --- a/src/libsyntax_ext/source_util.rs +++ b/src/libsyntax_ext/source_util.rs @@ -1,10 +1,11 @@ use syntax_expand::panictry; use syntax_expand::base::{self, *}; use syntax::ast; -use syntax::parse::{self, token, DirectoryOwnership}; +use syntax::parse::{self, DirectoryOwnership}; use syntax::print::pprust; use syntax::ptr::P; use syntax::symbol::Symbol; +use syntax::token; use syntax::tokenstream::TokenStream; use syntax::early_buffered_lints::BufferedEarlyLintId; diff --git a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs index 927e2c0820e..6277e2e7ea9 100644 --- a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs +++ b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs @@ -16,7 +16,7 @@ use syntax::parse; use syntax::parse::PResult; use syntax::parse::new_parser_from_source_str; use syntax::parse::parser::Parser; -use syntax::parse::token; +use syntax::token; use syntax::ptr::P; use syntax::parse::parser::attr::*; use syntax::print::pprust; diff --git a/src/test/ui-fulldeps/auxiliary/roman-numerals.rs b/src/test/ui-fulldeps/auxiliary/roman-numerals.rs index 3524f449c74..520347faa15 100644 --- a/src/test/ui-fulldeps/auxiliary/roman-numerals.rs +++ b/src/test/ui-fulldeps/auxiliary/roman-numerals.rs @@ -15,7 +15,7 @@ extern crate syntax_pos; extern crate rustc; extern crate rustc_driver; -use syntax::parse::token::{self, Token}; +use syntax::token::{self, Token}; use syntax::tokenstream::{TokenTree, TokenStream}; use syntax_expand::base::{ExtCtxt, MacResult, DummyResult, MacEager}; use syntax_pos::Span; -- cgit 1.4.1-3-g733a5 From be023ebe850261c6bb202a02a686827d821c3697 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 10 Oct 2019 10:26:10 +0200 Subject: move config.rs to libsyntax_expand --- src/librustc/session/mod.rs | 16 +- src/librustc_interface/interface.rs | 7 +- src/librustc_interface/passes.rs | 2 +- src/librustc_interface/tests.rs | 59 +- src/librustc_interface/util.rs | 2 + src/librustdoc/html/highlight.rs | 3 +- src/librustdoc/passes/check_code_block_syntax.rs | 3 +- src/librustdoc/test.rs | 3 +- src/libsyntax/attr/mod.rs | 6 +- src/libsyntax/config.rs | 352 ------ src/libsyntax/json/tests.rs | 15 +- src/libsyntax/lib.rs | 4 - src/libsyntax/mut_visit.rs | 3 - src/libsyntax/mut_visit/tests.rs | 71 -- src/libsyntax/parse/lexer/mod.rs | 6 +- src/libsyntax/parse/lexer/tests.rs | 270 ----- src/libsyntax/parse/mod.rs | 3 +- src/libsyntax/parse/parser/attr.rs | 2 +- src/libsyntax/parse/parser/module.rs | 19 +- src/libsyntax/parse/tests.rs | 339 ------ src/libsyntax/print/pprust.rs | 7 +- src/libsyntax/sess.rs | 29 +- src/libsyntax/tests.rs | 1255 -------------------- src/libsyntax/tokenstream.rs | 5 +- src/libsyntax/tokenstream/tests.rs | 108 -- src/libsyntax/util/comments.rs | 3 +- src/libsyntax_expand/config.rs | 365 ++++++ src/libsyntax_expand/expand.rs | 4 +- src/libsyntax_expand/lib.rs | 29 + src/libsyntax_expand/mut_visit/tests.rs | 70 ++ src/libsyntax_expand/parse/lexer/tests.rs | 277 +++++ src/libsyntax_expand/parse/tests.rs | 338 ++++++ src/libsyntax_expand/tests.rs | 1254 +++++++++++++++++++ src/libsyntax_expand/tokenstream/tests.rs | 110 ++ src/test/ui-fulldeps/ast_stmt_expr_attr.rs | 21 +- src/test/ui-fulldeps/mod_dir_path_canonicalized.rs | 8 +- src/test/ui-fulldeps/pprust-expr-roundtrip.rs | 8 +- 37 files changed, 2580 insertions(+), 2496 deletions(-) delete mode 100644 src/libsyntax/config.rs delete mode 100644 src/libsyntax/mut_visit/tests.rs delete mode 100644 src/libsyntax/parse/lexer/tests.rs delete mode 100644 src/libsyntax/parse/tests.rs delete mode 100644 src/libsyntax/tests.rs delete mode 100644 src/libsyntax/tokenstream/tests.rs create mode 100644 src/libsyntax_expand/config.rs create mode 100644 src/libsyntax_expand/mut_visit/tests.rs create mode 100644 src/libsyntax_expand/parse/lexer/tests.rs create mode 100644 src/libsyntax_expand/parse/tests.rs create mode 100644 src/libsyntax_expand/tests.rs create mode 100644 src/libsyntax_expand/tokenstream/tests.rs (limited to 'src/libsyntax/parse/tests.rs') diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index 403b32df20e..9d702e7d6bf 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -27,7 +27,7 @@ use syntax::expand::allocator::AllocatorKind; use syntax::feature_gate::{self, AttributeType}; use syntax::json::JsonEmitter; use syntax::source_map; -use syntax::sess::ParseSess; +use syntax::sess::{ParseSess, ProcessCfgMod}; use syntax::symbol::Symbol; use syntax_pos::{MultiSpan, Span}; use crate::util::profiling::{SelfProfiler, SelfProfilerRef}; @@ -952,6 +952,7 @@ pub fn build_session( sopts: config::Options, local_crate_source_file: Option, registry: errors::registry::Registry, + process_cfg_mod: ProcessCfgMod, ) -> Session { let file_path_mapping = sopts.file_path_mapping(); @@ -962,6 +963,7 @@ pub fn build_session( Lrc::new(source_map::SourceMap::new(file_path_mapping)), DiagnosticOutput::Default, Default::default(), + process_cfg_mod, ) } @@ -1040,6 +1042,7 @@ pub fn build_session_with_source_map( source_map: Lrc, diagnostics_output: DiagnosticOutput, lint_caps: FxHashMap, + process_cfg_mod: ProcessCfgMod, ) -> Session { // FIXME: This is not general enough to make the warning lint completely override // normal diagnostic warnings, since the warning lint can also be denied and changed @@ -1080,7 +1083,14 @@ pub fn build_session_with_source_map( }, ); - build_session_(sopts, local_crate_source_file, diagnostic_handler, source_map, lint_caps) + build_session_( + sopts, + local_crate_source_file, + diagnostic_handler, + source_map, + lint_caps, + process_cfg_mod, + ) } fn build_session_( @@ -1089,6 +1099,7 @@ fn build_session_( span_diagnostic: errors::Handler, source_map: Lrc, driver_lint_caps: FxHashMap, + process_cfg_mod: ProcessCfgMod, ) -> Session { let self_profiler = if let SwitchWithOptPath::Enabled(ref d) = sopts.debugging_opts.self_profile { @@ -1127,6 +1138,7 @@ fn build_session_( let parse_sess = ParseSess::with_span_handler( span_diagnostic, source_map, + process_cfg_mod, ); let sysroot = match &sopts.maybe_sysroot { Some(sysroot) => sysroot.clone(), diff --git a/src/librustc_interface/interface.rs b/src/librustc_interface/interface.rs index 034e861b212..61f30392e06 100644 --- a/src/librustc_interface/interface.rs +++ b/src/librustc_interface/interface.rs @@ -14,11 +14,12 @@ use rustc_data_structures::fx::{FxHashSet, FxHashMap}; use std::path::PathBuf; use std::result; use std::sync::{Arc, Mutex}; -use syntax::{self, parse}; use syntax::ast::{self, MetaItemKind}; +use syntax::parse::new_parser_from_source_str; use syntax::token; use syntax::source_map::{FileName, FileLoader, SourceMap}; use syntax::sess::ParseSess; +use syntax_expand::config::process_configure_mod; use syntax_pos::edition; pub type Result = result::Result; @@ -64,9 +65,9 @@ impl Compiler { pub fn parse_cfgspecs(cfgspecs: Vec) -> FxHashSet<(String, Option)> { syntax::with_default_globals(move || { let cfg = cfgspecs.into_iter().map(|s| { - let sess = ParseSess::with_silent_emitter(); + let sess = ParseSess::with_silent_emitter(process_configure_mod); let filename = FileName::cfg_spec_source_code(&s); - let mut parser = parse::new_parser_from_source_str(&sess, filename, s.to_string()); + let mut parser = new_parser_from_source_str(&sess, filename, s.to_string()); macro_rules! error {($reason: expr) => { early_error(ErrorOutputType::default(), diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index ce34caee6fa..c874e94124d 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -181,7 +181,7 @@ pub fn register_plugins<'a>( ) }); - let (krate, features) = syntax::config::features( + let (krate, features) = syntax_expand::config::features( krate, &sess.parse_sess, sess.edition(), diff --git a/src/librustc_interface/tests.rs b/src/librustc_interface/tests.rs index 7a57605da58..8c1dac21576 100644 --- a/src/librustc_interface/tests.rs +++ b/src/librustc_interface/tests.rs @@ -8,7 +8,7 @@ use rustc::session::config::{build_configuration, build_session_options, to_crat use rustc::session::config::{LtoCli, LinkerPluginLto, SwitchWithOptPath, ExternEntry}; use rustc::session::config::{Externs, OutputType, OutputTypes, SymbolManglingVersion}; use rustc::session::config::{rustc_optgroups, Options, ErrorOutputType, Passes}; -use rustc::session::build_session; +use rustc::session::{build_session, Session}; use rustc::session::search_paths::SearchPath; use std::collections::{BTreeMap, BTreeSet}; use std::iter::FromIterator; @@ -17,16 +17,23 @@ use rustc_target::spec::{MergeFunctions, PanicStrategy, RelroLevel}; use syntax::symbol::sym; use syntax::edition::{Edition, DEFAULT_EDITION}; use syntax; +use syntax_expand::config::process_configure_mod; use rustc_data_structures::fx::FxHashSet; use rustc_errors::{ColorConfig, emitter::HumanReadableErrorType, registry}; -pub fn build_session_options_and_crate_config( - matches: &getopts::Matches, -) -> (Options, FxHashSet<(String, Option)>) { - ( - build_session_options(matches), - parse_cfgspecs(matches.opt_strs("cfg")), - ) +type CfgSpecs = FxHashSet<(String, Option)>; + +fn build_session_options_and_crate_config(matches: getopts::Matches) -> (Options, CfgSpecs) { + let sessopts = build_session_options(&matches); + let cfg = parse_cfgspecs(matches.opt_strs("cfg")); + (sessopts, cfg) +} + +fn mk_session(matches: getopts::Matches) -> (Session, CfgSpecs) { + let registry = registry::Registry::new(&[]); + let (sessopts, cfg) = build_session_options_and_crate_config(matches); + let sess = build_session(sessopts, None, registry, process_configure_mod); + (sess, cfg) } fn new_public_extern_entry(locations: I) -> ExternEntry @@ -59,31 +66,19 @@ fn mk_map(entries: Vec<(K, V)>) -> BTreeMap { #[test] fn test_switch_implies_cfg_test() { syntax::with_default_globals(|| { - let matches = &match optgroups().parse(&["--test".to_string()]) { - Ok(m) => m, - Err(f) => panic!("test_switch_implies_cfg_test: {}", f), - }; - let registry = registry::Registry::new(&[]); - let (sessopts, cfg) = build_session_options_and_crate_config(matches); - let sess = build_session(sessopts, None, registry); + let matches = optgroups().parse(&["--test".to_string()]).unwrap(); + let (sess, cfg) = mk_session(matches); let cfg = build_configuration(&sess, to_crate_config(cfg)); assert!(cfg.contains(&(sym::test, None))); }); } -// When the user supplies --test and --cfg test, don't implicitly add -// another --cfg test +// When the user supplies --test and --cfg test, don't implicitly add another --cfg test #[test] fn test_switch_implies_cfg_test_unless_cfg_test() { syntax::with_default_globals(|| { - let matches = &match optgroups().parse(&["--test".to_string(), - "--cfg=test".to_string()]) { - Ok(m) => m, - Err(f) => panic!("test_switch_implies_cfg_test_unless_cfg_test: {}", f), - }; - let registry = registry::Registry::new(&[]); - let (sessopts, cfg) = build_session_options_and_crate_config(matches); - let sess = build_session(sessopts, None, registry); + let matches = optgroups().parse(&["--test".to_string(), "--cfg=test".to_string()]).unwrap(); + let (sess, cfg) = mk_session(matches); let cfg = build_configuration(&sess, to_crate_config(cfg)); let mut test_items = cfg.iter().filter(|&&(name, _)| name == sym::test); assert!(test_items.next().is_some()); @@ -95,9 +90,7 @@ fn test_switch_implies_cfg_test_unless_cfg_test() { fn test_can_print_warnings() { syntax::with_default_globals(|| { let matches = optgroups().parse(&["-Awarnings".to_string()]).unwrap(); - let registry = registry::Registry::new(&[]); - let (sessopts, _) = build_session_options_and_crate_config(&matches); - let sess = build_session(sessopts, None, registry); + let (sess, _) = mk_session(matches); assert!(!sess.diagnostic().can_emit_warnings()); }); @@ -105,17 +98,13 @@ fn test_can_print_warnings() { let matches = optgroups() .parse(&["-Awarnings".to_string(), "-Dwarnings".to_string()]) .unwrap(); - let registry = registry::Registry::new(&[]); - let (sessopts, _) = build_session_options_and_crate_config(&matches); - let sess = build_session(sessopts, None, registry); + let (sess, _) = mk_session(matches); assert!(sess.diagnostic().can_emit_warnings()); }); syntax::with_default_globals(|| { let matches = optgroups().parse(&["-Adead_code".to_string()]).unwrap(); - let registry = registry::Registry::new(&[]); - let (sessopts, _) = build_session_options_and_crate_config(&matches); - let sess = build_session(sessopts, None, registry); + let (sess, _) = mk_session(matches); assert!(sess.diagnostic().can_emit_warnings()); }); } @@ -704,6 +693,6 @@ fn test_edition_parsing() { let matches = optgroups() .parse(&["--edition=2018".to_string()]) .unwrap(); - let (sessopts, _) = build_session_options_and_crate_config(&matches); + let (sessopts, _) = build_session_options_and_crate_config(matches); assert!(sessopts.edition == Edition::Edition2018) } diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs index a74ea4bca39..c02e5b9ae28 100644 --- a/src/librustc_interface/util.rs +++ b/src/librustc_interface/util.rs @@ -36,6 +36,7 @@ use syntax::util::lev_distance::find_best_match_for_name; use syntax::source_map::{FileLoader, RealFileLoader, SourceMap}; use syntax::symbol::{Symbol, sym}; use syntax::{self, ast, attr}; +use syntax_expand::config::process_configure_mod; use syntax_pos::edition::Edition; #[cfg(not(parallel_compiler))] use std::{thread, panic}; @@ -103,6 +104,7 @@ pub fn create_session( source_map.clone(), diagnostic_output, lint_caps, + process_configure_mod, ); let codegen_backend = get_codegen_backend(&sess); diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 4bd72f7e61c..71a44c0ebdb 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -16,6 +16,7 @@ use syntax::parse::lexer; use syntax::token::{self, Token}; use syntax::sess::ParseSess; use syntax::symbol::{kw, sym}; +use syntax_expand::config::process_configure_mod; use syntax_pos::{Span, FileName}; /// Highlights `src`, returning the HTML output. @@ -33,7 +34,7 @@ pub fn render_with_highlighting( class, tooltip).unwrap(); } - let sess = ParseSess::with_silent_emitter(); + let sess = ParseSess::with_silent_emitter(process_configure_mod); let fm = sess.source_map().new_source_file( FileName::Custom(String::from("rustdoc-highlighting")), src.to_owned(), diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs index 4603e77b0fd..405f3a70e0e 100644 --- a/src/librustdoc/passes/check_code_block_syntax.rs +++ b/src/librustdoc/passes/check_code_block_syntax.rs @@ -3,6 +3,7 @@ use syntax::parse::lexer::{StringReader as Lexer}; use syntax::token; use syntax::sess::ParseSess; use syntax::source_map::FilePathMapping; +use syntax_expand::config::process_configure_mod; use syntax_pos::{InnerSpan, FileName}; use crate::clean; @@ -27,7 +28,7 @@ struct SyntaxChecker<'a, 'tcx> { impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { fn check_rust_syntax(&self, item: &clean::Item, dox: &str, code_block: RustCodeBlock) { - let sess = ParseSess::new(FilePathMapping::empty()); + let sess = ParseSess::new(FilePathMapping::empty(), process_configure_mod); let source_file = sess.source_map().new_source_file( FileName::Custom(String::from("doctest")), dox[code_block.code].to_owned(), diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index 07dc1e4e915..e4cf1bb0bb9 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -17,6 +17,7 @@ use std::path::PathBuf; use std::process::{self, Command, Stdio}; use std::str; use syntax::symbol::sym; +use syntax_expand::config::process_configure_mod; use syntax_pos::{BytePos, DUMMY_SP, Pos, Span, FileName}; use tempfile::Builder as TempFileBuilder; use testing; @@ -411,7 +412,7 @@ pub fn make_test(s: &str, let emitter = EmitterWriter::new(box io::sink(), None, false, false, false, None, false); // FIXME(misdreavus): pass `-Z treat-err-as-bug` to the doctest parser let handler = Handler::with_emitter(false, None, box emitter); - let sess = ParseSess::with_span_handler(handler, cm); + let sess = ParseSess::with_span_handler(handler, cm, process_configure_mod); let mut found_main = false; let mut found_extern_crate = cratename.is_none(); diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 2c011b06470..d458549e298 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -363,8 +363,12 @@ crate fn mk_attr_id() -> AttrId { } pub fn mk_attr(style: AttrStyle, path: Path, tokens: TokenStream, span: Span) -> Attribute { + mk_attr_from_item(style, AttrItem { path, tokens }, span) +} + +pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute { Attribute { - kind: AttrKind::Normal(AttrItem { path, tokens }), + kind: AttrKind::Normal(item), id: mk_attr_id(), style, span, diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs deleted file mode 100644 index a460fc2b99a..00000000000 --- a/src/libsyntax/config.rs +++ /dev/null @@ -1,352 +0,0 @@ -use crate::attr::HasAttrs; -use crate::feature_gate::{ - feature_err, - EXPLAIN_STMT_ATTR_SYNTAX, - Features, - get_features, - GateIssue, -}; -use crate::attr; -use crate::ast; -use crate::edition::Edition; -use crate::mut_visit::*; -use crate::parse::{self, validate_attr}; -use crate::ptr::P; -use crate::sess::ParseSess; -use crate::symbol::sym; -use crate::util::map_in_place::MapInPlace; - -use errors::Applicability; -use smallvec::SmallVec; - -/// A folder that strips out items that do not belong in the current configuration. -pub struct StripUnconfigured<'a> { - pub sess: &'a ParseSess, - pub features: Option<&'a Features>, -} - -// `cfg_attr`-process the crate's attributes and compute the crate's features. -pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, - allow_features: &Option>) -> (ast::Crate, Features) { - let features; - { - let mut strip_unconfigured = StripUnconfigured { - sess, - features: None, - }; - - let unconfigured_attrs = krate.attrs.clone(); - let err_count = sess.span_diagnostic.err_count(); - if let Some(attrs) = strip_unconfigured.configure(krate.attrs) { - krate.attrs = attrs; - } else { // the entire crate is unconfigured - krate.attrs = Vec::new(); - krate.module.items = Vec::new(); - return (krate, Features::new()); - } - - features = get_features(&sess.span_diagnostic, &krate.attrs, edition, allow_features); - - // Avoid reconfiguring malformed `cfg_attr`s - if err_count == sess.span_diagnostic.err_count() { - strip_unconfigured.features = Some(&features); - strip_unconfigured.configure(unconfigured_attrs); - } - } - - (krate, features) -} - -#[macro_export] -macro_rules! configure { - ($this:ident, $node:ident) => { - match $this.configure($node) { - Some(node) => node, - None => return Default::default(), - } - } -} - -impl<'a> StripUnconfigured<'a> { - pub fn configure(&mut self, mut node: T) -> Option { - self.process_cfg_attrs(&mut node); - if self.in_cfg(node.attrs()) { Some(node) } else { None } - } - - /// Parse and expand all `cfg_attr` attributes into a list of attributes - /// that are within each `cfg_attr` that has a true configuration predicate. - /// - /// Gives compiler warnigns if any `cfg_attr` does not contain any - /// attributes and is in the original source code. Gives compiler errors if - /// the syntax of any `cfg_attr` is incorrect. - pub fn process_cfg_attrs(&mut self, node: &mut T) { - node.visit_attrs(|attrs| { - attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr)); - }); - } - - /// Parse and expand a single `cfg_attr` attribute into a list of attributes - /// when the configuration predicate is true, or otherwise expand into an - /// empty list of attributes. - /// - /// Gives a compiler warning when the `cfg_attr` contains no attributes and - /// is in the original source file. Gives a compiler error if the syntax of - /// the attribute is incorrect. - fn process_cfg_attr(&mut self, attr: ast::Attribute) -> Vec { - if !attr.has_name(sym::cfg_attr) { - return vec![attr]; - } - if attr.get_normal_item().tokens.is_empty() { - self.sess.span_diagnostic - .struct_span_err( - attr.span, - "malformed `cfg_attr` attribute input", - ).span_suggestion( - attr.span, - "missing condition and attribute", - "#[cfg_attr(condition, attribute, other_attribute, ...)]".to_owned(), - Applicability::HasPlaceholders, - ).note("for more information, visit \ - ") - .emit(); - return vec![]; - } - - let res = parse::parse_in_attr(self.sess, &attr, |p| p.parse_cfg_attr()); - let (cfg_predicate, expanded_attrs) = match res { - Ok(result) => result, - Err(mut e) => { - e.emit(); - return vec![]; - } - }; - - // Lint on zero attributes in source. - if expanded_attrs.is_empty() { - return vec![attr]; - } - - // At this point we know the attribute is considered used. - attr::mark_used(&attr); - - if attr::cfg_matches(&cfg_predicate, self.sess, self.features) { - // We call `process_cfg_attr` recursively in case there's a - // `cfg_attr` inside of another `cfg_attr`. E.g. - // `#[cfg_attr(false, cfg_attr(true, some_attr))]`. - expanded_attrs.into_iter() - .flat_map(|(item, span)| self.process_cfg_attr(ast::Attribute { - kind: ast::AttrKind::Normal(item), - id: attr::mk_attr_id(), - style: attr.style, - span, - })) - .collect() - } else { - vec![] - } - } - - /// Determines if a node with the given attributes should be included in this configuration. - pub fn in_cfg(&mut self, attrs: &[ast::Attribute]) -> bool { - attrs.iter().all(|attr| { - if !is_cfg(attr) { - return true; - } - - let error = |span, msg, suggestion: &str| { - let mut err = self.sess.span_diagnostic.struct_span_err(span, msg); - if !suggestion.is_empty() { - err.span_suggestion( - span, - "expected syntax is", - suggestion.into(), - Applicability::MaybeIncorrect, - ); - } - err.emit(); - true - }; - - let meta_item = match validate_attr::parse_meta(self.sess, attr) { - Ok(meta_item) => meta_item, - Err(mut err) => { err.emit(); return true; } - }; - let nested_meta_items = if let Some(nested_meta_items) = meta_item.meta_item_list() { - nested_meta_items - } else { - return error(meta_item.span, "`cfg` is not followed by parentheses", - "cfg(/* predicate */)"); - }; - - if nested_meta_items.is_empty() { - return error(meta_item.span, "`cfg` predicate is not specified", ""); - } else if nested_meta_items.len() > 1 { - return error(nested_meta_items.last().unwrap().span(), - "multiple `cfg` predicates are specified", ""); - } - - match nested_meta_items[0].meta_item() { - Some(meta_item) => attr::cfg_matches(meta_item, self.sess, self.features), - None => error(nested_meta_items[0].span(), - "`cfg` predicate key cannot be a literal", ""), - } - }) - } - - /// Visit attributes on expression and statements (but not attributes on items in blocks). - fn visit_expr_attrs(&mut self, attrs: &[ast::Attribute]) { - // flag the offending attributes - for attr in attrs.iter() { - self.maybe_emit_expr_attr_err(attr); - } - } - - /// If attributes are not allowed on expressions, emit an error for `attr` - pub fn maybe_emit_expr_attr_err(&self, attr: &ast::Attribute) { - if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) { - let mut err = feature_err(self.sess, - sym::stmt_expr_attributes, - attr.span, - GateIssue::Language, - EXPLAIN_STMT_ATTR_SYNTAX); - - if attr.is_doc_comment() { - err.help("`///` is for documentation comments. For a plain comment, use `//`."); - } - - err.emit(); - } - } - - pub fn configure_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - let ast::ForeignMod { abi: _, items } = foreign_mod; - items.flat_map_in_place(|item| self.configure(item)); - } - - pub fn configure_generic_params(&mut self, params: &mut Vec) { - params.flat_map_in_place(|param| self.configure(param)); - } - - fn configure_variant_data(&mut self, vdata: &mut ast::VariantData) { - match vdata { - ast::VariantData::Struct(fields, ..) | ast::VariantData::Tuple(fields, _) => - fields.flat_map_in_place(|field| self.configure(field)), - ast::VariantData::Unit(_) => {} - } - } - - pub fn configure_item_kind(&mut self, item: &mut ast::ItemKind) { - match item { - ast::ItemKind::Struct(def, _generics) | - ast::ItemKind::Union(def, _generics) => self.configure_variant_data(def), - ast::ItemKind::Enum(ast::EnumDef { variants }, _generics) => { - variants.flat_map_in_place(|variant| self.configure(variant)); - for variant in variants { - self.configure_variant_data(&mut variant.data); - } - } - _ => {} - } - } - - pub fn configure_expr_kind(&mut self, expr_kind: &mut ast::ExprKind) { - match expr_kind { - ast::ExprKind::Match(_m, arms) => { - arms.flat_map_in_place(|arm| self.configure(arm)); - } - ast::ExprKind::Struct(_path, fields, _base) => { - fields.flat_map_in_place(|field| self.configure(field)); - } - _ => {} - } - } - - pub fn configure_expr(&mut self, expr: &mut P) { - self.visit_expr_attrs(expr.attrs()); - - // If an expr is valid to cfg away it will have been removed by the - // outer stmt or expression folder before descending in here. - // Anything else is always required, and thus has to error out - // in case of a cfg attr. - // - // N.B., this is intentionally not part of the visit_expr() function - // in order for filter_map_expr() to be able to avoid this check - if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a)) { - let msg = "removing an expression is not supported in this position"; - self.sess.span_diagnostic.span_err(attr.span, msg); - } - - self.process_cfg_attrs(expr) - } - - pub fn configure_pat(&mut self, pat: &mut P) { - if let ast::PatKind::Struct(_path, fields, _etc) = &mut pat.kind { - fields.flat_map_in_place(|field| self.configure(field)); - } - } - - pub fn configure_fn_decl(&mut self, fn_decl: &mut ast::FnDecl) { - fn_decl.inputs.flat_map_in_place(|arg| self.configure(arg)); - } -} - -impl<'a> MutVisitor for StripUnconfigured<'a> { - fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { - self.configure_foreign_mod(foreign_mod); - noop_visit_foreign_mod(foreign_mod, self); - } - - fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { - self.configure_item_kind(item); - noop_visit_item_kind(item, self); - } - - fn visit_expr(&mut self, expr: &mut P) { - self.configure_expr(expr); - self.configure_expr_kind(&mut expr.kind); - noop_visit_expr(expr, self); - } - - fn filter_map_expr(&mut self, expr: P) -> Option> { - let mut expr = configure!(self, expr); - self.configure_expr_kind(&mut expr.kind); - noop_visit_expr(&mut expr, self); - Some(expr) - } - - fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { - noop_flat_map_stmt(configure!(self, stmt), self) - } - - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { - noop_flat_map_item(configure!(self, item), self) - } - - fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { - noop_flat_map_impl_item(configure!(self, item), self) - } - - fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { - noop_flat_map_trait_item(configure!(self, item), self) - } - - fn visit_mac(&mut self, _mac: &mut ast::Mac) { - // Don't configure interpolated AST (cf. issue #34171). - // Interpolated AST will get configured once the surrounding tokens are parsed. - } - - fn visit_pat(&mut self, pat: &mut P) { - self.configure_pat(pat); - noop_visit_pat(pat, self) - } - - fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { - self.configure_fn_decl(&mut fn_decl); - noop_visit_fn_decl(fn_decl, self); - } -} - -fn is_cfg(attr: &ast::Attribute) -> bool { - attr.check_name(sym::cfg) -} diff --git a/src/libsyntax/json/tests.rs b/src/libsyntax/json/tests.rs index eb0d9ef3947..1edefd5bc4b 100644 --- a/src/libsyntax/json/tests.rs +++ b/src/libsyntax/json/tests.rs @@ -2,7 +2,6 @@ use super::*; use crate::json::JsonEmitter; use crate::source_map::{FilePathMapping, SourceMap}; -use crate::tests::Shared; use crate::with_default_globals; use errors::emitter::{ColorConfig, HumanReadableErrorType}; @@ -27,6 +26,20 @@ struct SpanTestData { pub column_end: u32, } +struct Shared { + data: Arc>, +} + +impl Write for Shared { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.data.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.data.lock().unwrap().flush() + } +} + /// Test the span yields correct positions in JSON. fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) { let expected_output = TestData { spans: vec![expected_output] }; diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 1b17de529c4..7fecc87a18f 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -26,9 +26,6 @@ pub use rustc_data_structures::thin_vec::ThinVec; use ast::AttrId; use syntax_pos::edition::Edition; -#[cfg(test)] -mod tests; - pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); #[macro_export] @@ -100,7 +97,6 @@ pub mod ast; pub mod attr; pub mod expand; pub mod source_map; -#[macro_use] pub mod config; pub mod entry; pub mod feature_gate; pub mod mut_visit; diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 7696ea48f93..376323a83ea 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -22,9 +22,6 @@ use rustc_data_structures::sync::Lrc; use std::ops::DerefMut; use std::{panic, process, ptr}; -#[cfg(test)] -mod tests; - pub trait ExpectOne { fn expect_one(self, err: &'static str) -> A::Item; } diff --git a/src/libsyntax/mut_visit/tests.rs b/src/libsyntax/mut_visit/tests.rs deleted file mode 100644 index f779e0d0a60..00000000000 --- a/src/libsyntax/mut_visit/tests.rs +++ /dev/null @@ -1,71 +0,0 @@ -use super::*; - -use crate::ast::{self, Ident}; -use crate::tests::{string_to_crate, matches_codepattern}; -use crate::print::pprust; -use crate::mut_visit; -use crate::with_default_globals; - -// This version doesn't care about getting comments or doc-strings in. -fn fake_print_crate(s: &mut pprust::State<'_>, - krate: &ast::Crate) { - s.print_mod(&krate.module, &krate.attrs) -} - -// Change every identifier to "zz". -struct ToZzIdentMutVisitor; - -impl MutVisitor for ToZzIdentMutVisitor { - fn visit_ident(&mut self, ident: &mut ast::Ident) { - *ident = Ident::from_str("zz"); - } - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } -} - -// Maybe add to `expand.rs`. -macro_rules! assert_pred { - ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( - { - let pred_val = $pred; - let a_val = $a; - let b_val = $b; - if !(pred_val(&a_val, &b_val)) { - panic!("expected args satisfying {}, got {} and {}", - $predname, a_val, b_val); - } - } - ) -} - -// Make sure idents get transformed everywhere. -#[test] fn ident_transformation () { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); - }) -} - -// Make sure idents get transformed even inside macro defs. -#[test] fn ident_transformation_in_defs () { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ - (g $(d $d $e)+))} ".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string()); - }) -} diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index b1b7b08c78a..f2d5ff3440e 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -13,9 +13,6 @@ use std::convert::TryInto; use rustc_data_structures::sync::Lrc; use log::debug; -#[cfg(test)] -mod tests; - mod tokentrees; mod unicode_chars; mod unescape_error_reporting; @@ -35,7 +32,8 @@ pub struct StringReader<'a> { /// Initial position, read-only. start_pos: BytePos, /// The absolute offset within the source_map of the current character. - pos: BytePos, + // FIXME(#64197): `pub` is needed by tests for now. + pub pos: BytePos, /// Stop reading src at this index. end_src_index: usize, /// Source text to tokenize. diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs deleted file mode 100644 index baa6fb59537..00000000000 --- a/src/libsyntax/parse/lexer/tests.rs +++ /dev/null @@ -1,270 +0,0 @@ -use super::*; - -use crate::symbol::Symbol; -use crate::source_map::{SourceMap, FilePathMapping}; -use crate::token; -use crate::util::comments::is_doc_comment; -use crate::with_default_globals; - -use errors::{Handler, emitter::EmitterWriter}; -use std::io; -use std::path::PathBuf; -use syntax_pos::{BytePos, Span}; - -fn mk_sess(sm: Lrc) -> ParseSess { - let emitter = EmitterWriter::new( - Box::new(io::sink()), - Some(sm.clone()), - false, - false, - false, - None, - false, - ); - ParseSess::with_span_handler(Handler::with_emitter(true, None, Box::new(emitter)), sm) -} - -// Creates a string reader for the given string. -fn setup<'a>(sm: &SourceMap, - sess: &'a ParseSess, - teststr: String) - -> StringReader<'a> { - let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); - StringReader::new(sess, sf, None) -} - -#[test] -fn t1() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut string_reader = setup( - &sm, - &sh, - "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(), - ); - assert_eq!(string_reader.next_token(), token::Comment); - assert_eq!(string_reader.next_token(), token::Whitespace); - let tok1 = string_reader.next_token(); - let tok2 = Token::new( - mk_ident("fn"), - Span::with_root_ctxt(BytePos(21), BytePos(23)), - ); - assert_eq!(tok1.kind, tok2.kind); - assert_eq!(tok1.span, tok2.span); - assert_eq!(string_reader.next_token(), token::Whitespace); - // Read another token. - let tok3 = string_reader.next_token(); - assert_eq!(string_reader.pos.clone(), BytePos(28)); - let tok4 = Token::new( - mk_ident("main"), - Span::with_root_ctxt(BytePos(24), BytePos(28)), - ); - assert_eq!(tok3.kind, tok4.kind); - assert_eq!(tok3.span, tok4.span); - - assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); - assert_eq!(string_reader.pos.clone(), BytePos(29)) - }) -} - -// Checks that the given reader produces the desired stream -// of tokens (stop checking after exhausting `expected`). -fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec) { - for expected_tok in &expected { - assert_eq!(&string_reader.next_token(), expected_tok); - } -} - -// Makes the identifier by looking up the string in the interner. -fn mk_ident(id: &str) -> TokenKind { - token::Ident(Symbol::intern(id), false) -} - -fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { - TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) -} - -#[test] -fn doublecolon_parsing() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a b".to_string()), - vec![mk_ident("a"), token::Whitespace, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_2() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a::b".to_string()), - vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_3() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a ::b".to_string()), - vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")], - ); - }) -} - -#[test] -fn doublecolon_parsing_4() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization( - setup(&sm, &sh, "a:: b".to_string()), - vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")], - ); - }) -} - -#[test] -fn character_a() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "'a'".to_string()).next_token(), - mk_lit(token::Char, "a", None), - ); - }) -} - -#[test] -fn character_space() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "' '".to_string()).next_token(), - mk_lit(token::Char, " ", None), - ); - }) -} - -#[test] -fn character_escaped() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "'\\n'".to_string()).next_token(), - mk_lit(token::Char, "\\n", None), - ); - }) -} - -#[test] -fn lifetime_name() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "'abc".to_string()).next_token(), - token::Lifetime(Symbol::intern("'abc")), - ); - }) -} - -#[test] -fn raw_string() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!( - setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), - mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None), - ); - }) -} - -#[test] -fn literal_suffixes() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - macro_rules! test { - ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ - assert_eq!( - setup(&sm, &sh, format!("{}suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, Some("suffix")), - ); - // with a whitespace separator - assert_eq!( - setup(&sm, &sh, format!("{} suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, None), - ); - }} - } - - test!("'a'", Char, "a"); - test!("b'a'", Byte, "a"); - test!("\"a\"", Str, "a"); - test!("b\"a\"", ByteStr, "a"); - test!("1234", Integer, "1234"); - test!("0b101", Integer, "0b101"); - test!("0xABC", Integer, "0xABC"); - test!("1.0", Float, "1.0"); - test!("1.0e10", Float, "1.0e10"); - - assert_eq!( - setup(&sm, &sh, "2us".to_string()).next_token(), - mk_lit(token::Integer, "2", Some("us")), - ); - assert_eq!( - setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::StrRaw(3), "raw", Some("suffix")), - ); - assert_eq!( - setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")), - ); - }) -} - -#[test] -fn line_doc_comments() { - assert!(is_doc_comment("///")); - assert!(is_doc_comment("/// blah")); - assert!(!is_doc_comment("////")); -} - -#[test] -fn nested_block_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); - assert_eq!(lexer.next_token(), token::Comment); - assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); - }) -} - -#[test] -fn crlf_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); - let comment = lexer.next_token(); - assert_eq!(comment.kind, token::Comment); - assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); - assert_eq!(lexer.next_token(), token::Whitespace); - assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); - }) -} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 9155cbe5dd8..af3f8ecb45a 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -17,8 +17,7 @@ use std::str; use log::info; -#[cfg(test)] -mod tests; +pub const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); #[macro_use] pub mod parser; diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 31f0a02a483..0f9e573af82 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -268,7 +268,7 @@ impl<'a> Parser<'a> { } /// Parses `cfg_attr(pred, attr_item_list)` where `attr_item_list` is comma-delimited. - crate fn parse_cfg_attr(&mut self) -> PResult<'a, (ast::MetaItem, Vec<(ast::AttrItem, Span)>)> { + pub fn parse_cfg_attr(&mut self) -> PResult<'a, (ast::MetaItem, Vec<(ast::AttrItem, Span)>)> { self.expect(&token::OpenDelim(token::Paren))?; let cfg_predicate = self.parse_meta_item()?; diff --git a/src/libsyntax/parse/parser/module.rs b/src/libsyntax/parse/parser/module.rs index 3e5974c2eee..ad72b3a1dea 100644 --- a/src/libsyntax/parse/parser/module.rs +++ b/src/libsyntax/parse/parser/module.rs @@ -7,8 +7,8 @@ use crate::ast::{self, Ident, Attribute, ItemKind, Mod, Crate}; use crate::parse::{new_sub_parser_from_file, DirectoryOwnership}; use crate::token::{self, TokenKind}; use crate::source_map::{SourceMap, Span, DUMMY_SP, FileName}; -use crate::symbol::sym; +use syntax_pos::symbol::sym; use errors::PResult; use std::path::{self, Path, PathBuf}; @@ -39,17 +39,12 @@ impl<'a> Parser<'a> { /// Parses a `mod { ... }` or `mod ;` item. pub(super) fn parse_item_mod(&mut self, outer_attrs: &[Attribute]) -> PResult<'a, ItemInfo> { - let (in_cfg, outer_attrs) = { - // FIXME(Centril): This results in a cycle between config and parsing. - // Consider using dynamic dispatch via `self.sess` to disentangle the knot. - let mut strip_unconfigured = crate::config::StripUnconfigured { - sess: self.sess, - features: None, // Don't perform gated feature checking. - }; - let mut outer_attrs = outer_attrs.to_owned(); - strip_unconfigured.process_cfg_attrs(&mut outer_attrs); - (!self.cfg_mods || strip_unconfigured.in_cfg(&outer_attrs), outer_attrs) - }; + // HACK(Centril): See documentation on `ParseSess::process_cfg_mod`. + let (in_cfg, outer_attrs) = (self.sess.process_cfg_mod)( + self.sess, + self.cfg_mods, + outer_attrs, + ); let id_span = self.token.span; let id = self.parse_ident()?; diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs deleted file mode 100644 index 27ca2b6472f..00000000000 --- a/src/libsyntax/parse/tests.rs +++ /dev/null @@ -1,339 +0,0 @@ -use super::*; - -use crate::ast::{self, Name, PatKind}; -use crate::attr::first_attr_value_str_by_name; -use crate::sess::ParseSess; -use crate::parse::{PResult, new_parser_from_source_str}; -use crate::token::Token; -use crate::print::pprust::item_to_string; -use crate::ptr::P; -use crate::source_map::FilePathMapping; -use crate::symbol::{kw, sym}; -use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; -use crate::tokenstream::{DelimSpan, TokenTree, TokenStream}; -use crate::with_default_globals; -use syntax_pos::{Span, BytePos, Pos}; - -use std::path::PathBuf; - -/// Parses an item. -/// -/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` -/// when a syntax error occurred. -fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<'_, Option>> { - new_parser_from_source_str(sess, name, source).parse_item() -} - -// Produces a `syntax_pos::span`. -fn sp(a: u32, b: u32) -> Span { - Span::with_root_ctxt(BytePos(a), BytePos(b)) -} - -/// Parses a string, return an expression. -fn string_to_expr(source_str : String) -> P { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_expr() - }) -} - -/// Parses a string, returns an item. -fn string_to_item(source_str : String) -> Option> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_item() - }) -} - -#[should_panic] -#[test] fn bad_path_expr_1() { - with_default_globals(|| { - string_to_expr("::abc::def::return".to_string()); - }) -} - -// Checks the token-tree-ization of macros. -#[test] -fn string_to_tts_macro () { - with_default_globals(|| { - let tts: Vec<_> = - string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); - let tts: &[TokenTree] = &tts[..]; - - match tts { - [ - TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), - TokenTree::Token(Token { kind: token::Not, .. }), - TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), - TokenTree::Delimited(_, macro_delim, macro_tts) - ] - if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => { - let tts = ¯o_tts.trees().collect::>(); - match &tts[..] { - [ - TokenTree::Delimited(_, first_delim, first_tts), - TokenTree::Token(Token { kind: token::FatArrow, .. }), - TokenTree::Delimited(_, second_delim, second_tts), - ] - if macro_delim == &token::Paren => { - let tts = &first_tts.trees().collect::>(); - match &tts[..] { - [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), - ] - if first_delim == &token::Paren && name.as_str() == "a" => {}, - _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), - } - let tts = &second_tts.trees().collect::>(); - match &tts[..] { - [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), - ] - if second_delim == &token::Paren && name.as_str() == "a" => {}, - _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), - } - }, - _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), - } - }, - _ => panic!("value: {:?}",tts), - } - }) -} - -#[test] -fn string_to_tts_1() { - with_default_globals(|| { - let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - - let expected = TokenStream::new(vec![ - TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), - TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(5, 6), sp(13, 14)), - token::DelimToken::Paren, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), - TokenTree::token(token::Colon, sp(8, 9)).into(), - TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), - ]).into(), - ).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(15, 16), sp(20, 21)), - token::DelimToken::Brace, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), - TokenTree::token(token::Semi, sp(18, 19)).into(), - ]).into(), - ).into() - ]); - - assert_eq!(tts, expected); - }) -} - -#[test] fn parse_use() { - with_default_globals(|| { - let use_s = "use foo::bar::baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - - let use_s = "use foo::bar as baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - }) -} - -#[test] fn parse_extern_crate() { - with_default_globals(|| { - let ex_s = "extern crate foo;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - - let ex_s = "extern crate foo as bar;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - }) -} - -fn get_spans_of_pat_idents(src: &str) -> Vec { - let item = string_to_item(src.to_string()).unwrap(); - - struct PatIdentVisitor { - spans: Vec - } - impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { - fn visit_pat(&mut self, p: &'a ast::Pat) { - match p.kind { - PatKind::Ident(_ , ref ident, _) => { - self.spans.push(ident.span.clone()); - } - _ => { - crate::visit::walk_pat(self, p); - } - } - } - } - let mut v = PatIdentVisitor { spans: Vec::new() }; - crate::visit::walk_item(&mut v, &item); - return v.spans; -} - -#[test] fn span_of_self_arg_pat_idents_are_correct() { - with_default_globals(|| { - - let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", - "impl z { fn a (&mut self, &myarg: i32) {} }", - "impl z { fn a (&'a self, &myarg: i32) {} }", - "impl z { fn a (self, &myarg: i32) {} }", - "impl z { fn a (self: Foo, &myarg: i32) {} }", - ]; - - for &src in &srcs { - let spans = get_spans_of_pat_idents(src); - let (lo, hi) = (spans[0].lo(), spans[0].hi()); - assert!("self" == &src[lo.to_usize()..hi.to_usize()], - "\"{}\" != \"self\". src=\"{}\"", - &src[lo.to_usize()..hi.to_usize()], src) - } - }) -} - -#[test] fn parse_exprs () { - with_default_globals(|| { - // just make sure that they parse.... - string_to_expr("3 + 4".to_string()); - string_to_expr("a::z.froob(b,&(987+3))".to_string()); - }) -} - -#[test] fn attrs_fix_bug () { - with_default_globals(|| { - string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) - -> Result, String> { -#[cfg(windows)] -fn wb() -> c_int { - (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int -} - -#[cfg(unix)] -fn wb() -> c_int { O_WRONLY as c_int } - -let mut fflags: c_int = wb(); -}".to_string()); - }) -} - -#[test] fn crlf_doc_comments() { - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - - let name_1 = FileName::Custom("crlf_source_1".to_string()); - let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_1, source, &sess) - .unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); - assert_eq!(doc.as_str(), "/// doc comment"); - - let name_2 = FileName::Custom("crlf_source_2".to_string()); - let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_2, source, &sess) - .unwrap().unwrap(); - let docs = item.attrs.iter().filter(|a| a.has_name(sym::doc)) - .map(|a| a.value_str().unwrap().to_string()).collect::>(); - let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; - assert_eq!(&docs[..], b); - - let name_3 = FileName::Custom("clrf_source_3".to_string()); - let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); - assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); - }); -} - -#[test] -fn ttdelim_span() { - fn parse_expr_from_source_str( - name: FileName, source: String, sess: &ParseSess - ) -> PResult<'_, P> { - new_parser_from_source_str(sess, name, source).parse_expr() - } - - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), - "foo!( fn main() { body } )".to_string(), &sess).unwrap(); - - let tts: Vec<_> = match expr.kind { - ast::ExprKind::Mac(ref mac) => mac.stream().trees().collect(), - _ => panic!("not a macro"), - }; - - let span = tts.iter().rev().next().unwrap().span(); - - match sess.source_map().span_to_snippet(span) { - Ok(s) => assert_eq!(&s[..], "{ body }"), - Err(_) => panic!("could not get snippet"), - } - }); -} - -// This tests that when parsing a string (rather than a file) we don't try -// and read in a file for a module declaration and just parse a stub. -// See `recurse_into_file_modules` in the parser. -#[test] -fn out_of_line_mod() { - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let item = parse_item_from_source_str( - PathBuf::from("foo").into(), - "mod foo { struct S; mod this_does_not_exist; }".to_owned(), - &sess, - ).unwrap().unwrap(); - - if let ast::ItemKind::Mod(ref m) = item.kind { - assert!(m.items.len() == 2); - } else { - panic!(); - } - }); -} - -#[test] -fn eqmodws() { - assert_eq!(matches_codepattern("",""),true); - assert_eq!(matches_codepattern("","a"),false); - assert_eq!(matches_codepattern("a",""),false); - assert_eq!(matches_codepattern("a","a"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); - assert_eq!(matches_codepattern("a b","a b"),true); - assert_eq!(matches_codepattern("ab","a b"),false); - assert_eq!(matches_codepattern("a b","ab"),true); - assert_eq!(matches_codepattern(" a b","ab"),true); -} - -#[test] -fn pattern_whitespace() { - assert_eq!(matches_codepattern("","\x0C"), false); - assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); -} - -#[test] -fn non_pattern_whitespace() { - // These have the property 'White_Space' but not 'Pattern_White_Space' - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); - assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); -} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2203e8d9d06..04f096200b8 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -939,8 +939,11 @@ impl<'a> State<'a> { self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e), |e| e.span) } - crate fn print_mod(&mut self, _mod: &ast::Mod, - attrs: &[ast::Attribute]) { + pub fn print_mod( + &mut self, + _mod: &ast::Mod, + attrs: &[ast::Attribute], + ) { self.print_inner_attributes(attrs); for item in &_mod.items { self.print_item(item); diff --git a/src/libsyntax/sess.rs b/src/libsyntax/sess.rs index faad3e4af1e..5e96b27c970 100644 --- a/src/libsyntax/sess.rs +++ b/src/libsyntax/sess.rs @@ -1,7 +1,7 @@ //! Contains `ParseSess` which holds state living beyond what one `Parser` might. //! It also serves as an input to the parser itself. -use crate::ast::{CrateConfig, NodeId}; +use crate::ast::{CrateConfig, NodeId, Attribute}; use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId}; use crate::source_map::{SourceMap, FilePathMapping}; use crate::feature_gate::UnstableFeatures; @@ -89,10 +89,22 @@ pub struct ParseSess { pub gated_spans: GatedSpans, /// The parser has reached `Eof` due to an unclosed brace. Used to silence unnecessary errors. pub reached_eof: Lock, + /// Process the potential `cfg` attributes on a module. + /// Also determine if the module should be included in this configuration. + /// + /// HACK(Centril): This is used to break a cyclic dependency between + /// the parser and cfg-stripping as defined in `syntax_expand::config`. + /// The dependency edge from the parser comes from `parse_item_mod`. + /// A principled solution to this hack would be to implement [#64197]. + /// + /// [#64197]: https://github.com/rust-lang/rust/issues/64197 + pub process_cfg_mod: ProcessCfgMod, } +pub type ProcessCfgMod = fn(&ParseSess, bool, &[Attribute]) -> (bool, Vec); + impl ParseSess { - pub fn new(file_path_mapping: FilePathMapping) -> Self { + pub fn new(file_path_mapping: FilePathMapping, process_cfg_mod: ProcessCfgMod) -> Self { let cm = Lrc::new(SourceMap::new(file_path_mapping)); let handler = Handler::with_tty_emitter( ColorConfig::Auto, @@ -100,12 +112,17 @@ impl ParseSess { None, Some(cm.clone()), ); - ParseSess::with_span_handler(handler, cm) + ParseSess::with_span_handler(handler, cm, process_cfg_mod) } - pub fn with_span_handler(handler: Handler, source_map: Lrc) -> Self { + pub fn with_span_handler( + handler: Handler, + source_map: Lrc, + process_cfg_mod: ProcessCfgMod, + ) -> Self { Self { span_diagnostic: handler, + process_cfg_mod, unstable_features: UnstableFeatures::from_environment(), config: FxHashSet::default(), edition: ExpnId::root().expn_data().edition, @@ -121,10 +138,10 @@ impl ParseSess { } } - pub fn with_silent_emitter() -> Self { + pub fn with_silent_emitter(process_cfg_mod: ProcessCfgMod) -> Self { let cm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let handler = Handler::with_emitter(false, None, Box::new(SilentEmitter)); - ParseSess::with_span_handler(handler, cm) + ParseSess::with_span_handler(handler, cm, process_cfg_mod) } #[inline] diff --git a/src/libsyntax/tests.rs b/src/libsyntax/tests.rs deleted file mode 100644 index ed457c3627f..00000000000 --- a/src/libsyntax/tests.rs +++ /dev/null @@ -1,1255 +0,0 @@ -use crate::ast; -use crate::parse::source_file_to_stream; -use crate::parse::new_parser_from_source_str; -use crate::parse::parser::Parser; -use crate::sess::ParseSess; -use crate::source_map::{SourceMap, FilePathMapping}; -use crate::tokenstream::TokenStream; -use crate::with_default_globals; - -use errors::emitter::EmitterWriter; -use errors::{PResult, Handler}; -use rustc_data_structures::sync::Lrc; -use syntax_pos::{BytePos, Span, MultiSpan}; - -use std::io; -use std::io::prelude::*; -use std::iter::Peekable; -use std::path::{Path, PathBuf}; -use std::str; -use std::sync::{Arc, Mutex}; - -/// Map string to parser (via tts). -fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { - new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) -} - -crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where - F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, -{ - let mut p = string_to_parser(&ps, s); - let x = f(&mut p).unwrap(); - p.sess.span_diagnostic.abort_if_errors(); - x -} - -/// Maps a string to tts, using a made-up filename. -crate fn string_to_stream(source_str: String) -> TokenStream { - let ps = ParseSess::new(FilePathMapping::empty()); - source_file_to_stream( - &ps, - ps.source_map().new_source_file(PathBuf::from("bogofile").into(), - source_str, - ), None).0 -} - -/// Parses a string, returns a crate. -crate fn string_to_crate(source_str : String) -> ast::Crate { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_crate_mod() - }) -} - -/// Does the given string match the pattern? whitespace in the first string -/// may be deleted or replaced with other whitespace to match the pattern. -/// This function is relatively Unicode-ignorant; fortunately, the careful design -/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). -crate fn matches_codepattern(a : &str, b : &str) -> bool { - let mut a_iter = a.chars().peekable(); - let mut b_iter = b.chars().peekable(); - - loop { - let (a, b) = match (a_iter.peek(), b_iter.peek()) { - (None, None) => return true, - (None, _) => return false, - (Some(&a), None) => { - if rustc_lexer::is_whitespace(a) { - break // Trailing whitespace check is out of loop for borrowck. - } else { - return false - } - } - (Some(&a), Some(&b)) => (a, b) - }; - - if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) { - // Skip whitespace for `a` and `b`. - scan_for_non_ws_or_end(&mut a_iter); - scan_for_non_ws_or_end(&mut b_iter); - } else if rustc_lexer::is_whitespace(a) { - // Skip whitespace for `a`. - scan_for_non_ws_or_end(&mut a_iter); - } else if a == b { - a_iter.next(); - b_iter.next(); - } else { - return false - } - } - - // Check if a has *only* trailing whitespace. - a_iter.all(rustc_lexer::is_whitespace) -} - -/// Advances the given peekable `Iterator` until it reaches a non-whitespace character. -fn scan_for_non_ws_or_end>(iter: &mut Peekable) { - while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) { - iter.next(); - } -} - -/// Identifies a position in the text by the n'th occurrence of a string. -struct Position { - string: &'static str, - count: usize, -} - -struct SpanLabel { - start: Position, - end: Position, - label: &'static str, -} - -crate struct Shared { - pub data: Arc>, -} - -impl Write for Shared { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.data.lock().unwrap().write(buf) - } - - fn flush(&mut self) -> io::Result<()> { - self.data.lock().unwrap().flush() - } -} - -fn test_harness(file_text: &str, span_labels: Vec, expected_output: &str) { - with_default_globals(|| { - let output = Arc::new(Mutex::new(Vec::new())); - - let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); - source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned()); - - let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end); - let mut msp = MultiSpan::from_span(primary_span); - for span_label in span_labels { - let span = make_span(&file_text, &span_label.start, &span_label.end); - msp.push_span_label(span, span_label.label.to_string()); - println!("span: {:?} label: {:?}", span, span_label.label); - println!("text: {:?}", source_map.span_to_snippet(span)); - } - - let emitter = EmitterWriter::new( - Box::new(Shared { data: output.clone() }), - Some(source_map.clone()), - false, - false, - false, - None, - false, - ); - let handler = Handler::with_emitter(true, None, Box::new(emitter)); - handler.span_err(msp, "foo"); - - assert!(expected_output.chars().next() == Some('\n'), - "expected output should begin with newline"); - let expected_output = &expected_output[1..]; - - let bytes = output.lock().unwrap(); - let actual_output = str::from_utf8(&bytes).unwrap(); - println!("expected output:\n------\n{}------", expected_output); - println!("actual output:\n------\n{}------", actual_output); - - assert!(expected_output == actual_output) - }) -} - -fn make_span(file_text: &str, start: &Position, end: &Position) -> Span { - let start = make_pos(file_text, start); - let end = make_pos(file_text, end) + end.string.len(); // just after matching thing ends - assert!(start <= end); - Span::with_root_ctxt(BytePos(start as u32), BytePos(end as u32)) -} - -fn make_pos(file_text: &str, pos: &Position) -> usize { - let mut remainder = file_text; - let mut offset = 0; - for _ in 0..pos.count { - if let Some(n) = remainder.find(&pos.string) { - offset += n; - remainder = &remainder[n + 1..]; - } else { - panic!("failed to find {} instances of {:?} in {:?}", - pos.count, - pos.string, - file_text); - } - } - offset -} - -#[test] -fn ends_on_col0() { - test_harness(r#" -fn foo() { -} -"#, - vec![ - SpanLabel { - start: Position { - string: "{", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "test", - }, - ], - r#" -error: foo - --> test.rs:2:10 - | -2 | fn foo() { - | __________^ -3 | | } - | |_^ test - -"#); -} - -#[test] -fn ends_on_col2() { - test_harness(r#" -fn foo() { - - - } -"#, - vec![ - SpanLabel { - start: Position { - string: "{", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "test", - }, - ], - r#" -error: foo - --> test.rs:2:10 - | -2 | fn foo() { - | __________^ -3 | | -4 | | -5 | | } - | |___^ test - -"#); -} -#[test] -fn non_nested() { - test_harness(r#" -fn foo() { - X0 Y0 - X1 Y1 - X2 Y2 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "Y2", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 - | ____^__- - | | ___| - | || -4 | || X1 Y1 -5 | || X2 Y2 - | ||____^__- `Y` is a good letter too - | |____| - | `X` is a good letter - -"#); -} - -#[test] -fn nested() { - test_harness(r#" -fn foo() { - X0 Y0 - Y1 X1 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X1", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "Y1", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], -r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 - | ____^__- - | | ___| - | || -4 | || Y1 X1 - | ||____-__^ `X` is a good letter - | |_____| - | `Y` is a good letter too - -"#); -} - -#[test] -fn different_overlap() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Z1", - count: 1, - }, - end: Position { - string: "X3", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |_________- -5 | || X2 Y2 Z2 - | ||____^ `X` is a good letter -6 | | X3 Y3 Z3 - | |_____- `Y` is a good letter too - -"#); -} - -#[test] -fn triple_overlap() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "Y2", - count: 1, - }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { - string: "Z0", - count: 1, - }, - end: Position { - string: "Z2", - count: 1, - }, - label: "`Z` label", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | X0 Y0 Z0 - | _____^__-__- - | | ____|__| - | || ___| - | ||| -4 | ||| X1 Y1 Z1 -5 | ||| X2 Y2 Z2 - | |||____^__-__- `Z` label - | ||____|__| - | |____| `Y` is a good letter too - | `X` is a good letter - -"#); -} - -#[test] -fn triple_exact_overlap() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X2", - count: 1, - }, - label: "`Z` label", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | / X0 Y0 Z0 -4 | | X1 Y1 Z1 -5 | | X2 Y2 Z2 - | | ^ - | | | - | | `X` is a good letter - | |____`Y` is a good letter too - | `Z` label - -"#); -} - -#[test] -fn minimum_depth() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "X1", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Y1", - count: 1, - }, - end: Position { - string: "Z2", - count: 1, - }, - label: "`Y` is a good letter too", - }, - SpanLabel { - start: Position { - string: "X2", - count: 1, - }, - end: Position { - string: "Y3", - count: 1, - }, - label: "`Z`", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^_- - | ||____| - | | `X` is a good letter -5 | | X2 Y2 Z2 - | |____-______- `Y` is a good letter too - | ____| - | | -6 | | X3 Y3 Z3 - | |________- `Z` - -"#); -} - -#[test] -fn non_overlaping() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "X0", - count: 1, - }, - end: Position { - string: "X1", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Y2", - count: 1, - }, - end: Position { - string: "Z3", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | / X0 Y0 Z0 -4 | | X1 Y1 Z1 - | |____^ `X` is a good letter -5 | X2 Y2 Z2 - | ______- -6 | | X3 Y3 Z3 - | |__________- `Y` is a good letter too - -"#); -} - -#[test] -fn overlaping_start_and_end() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "X1", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Z1", - count: 1, - }, - end: Position { - string: "Z3", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^____- - | ||____| - | | `X` is a good letter -5 | | X2 Y2 Z2 -6 | | X3 Y3 Z3 - | |___________- `Y` is a good letter too - -"#); -} - -#[test] -fn multiple_labels_primary_without_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { - string: "c", - count: 1, - }, - end: Position { - string: "c", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- `a` is a good letter - -"#); -} - -#[test] -fn multiple_labels_secondary_without_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ `a` is a good letter - -"#); -} - -#[test] -fn multiple_labels_primary_without_message_2() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "`b` is a good letter", - }, - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "c", - count: 1, - }, - end: Position { - string: "c", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- - | | - | `b` is a good letter - -"#); -} - -#[test] -fn multiple_labels_secondary_without_message_2() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "`b` is a good letter", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - | | - | `b` is a good letter - -"#); -} - -#[test] -fn multiple_labels_secondary_without_message_3() { - test_harness(r#" -fn foo() { - a bc d -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "b", - count: 1, - }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { - string: "c", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a bc d - | ^^^^---- - | | - | `a` is a good letter - -"#); -} - -#[test] -fn multiple_labels_without_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - -"#); -} - -#[test] -fn multiple_labels_without_message_2() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - SpanLabel { - start: Position { - string: "c", - count: 1, - }, - end: Position { - string: "c", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:7 - | -3 | a { b { c } d } - | ----^^^^-^^-- - -"#); -} - -#[test] -fn multiple_labels_with_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "`a` is a good letter", - }, - SpanLabel { - start: Position { - string: "b", - count: 1, - }, - end: Position { - string: "}", - count: 1, - }, - label: "`b` is a good letter", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^-------^^ - | | | - | | `b` is a good letter - | `a` is a good letter - -"#); -} - -#[test] -fn single_label_with_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "`a` is a good letter", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^^^^^^^^^^ `a` is a good letter - -"#); -} - -#[test] -fn single_label_without_message() { - test_harness(r#" -fn foo() { - a { b { c } d } -} -"#, - vec![ - SpanLabel { - start: Position { - string: "a", - count: 1, - }, - end: Position { - string: "d", - count: 1, - }, - label: "", - }, - ], - r#" -error: foo - --> test.rs:3:3 - | -3 | a { b { c } d } - | ^^^^^^^^^^^^^ - -"#); -} - -#[test] -fn long_snippet() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 - X1 Y1 Z1 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 - X2 Y2 Z2 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "X1", - count: 1, - }, - label: "`X` is a good letter", - }, - SpanLabel { - start: Position { - string: "Z1", - count: 1, - }, - end: Position { - string: "Z3", - count: 1, - }, - label: "`Y` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | X1 Y1 Z1 - | |____^____- - | ||____| - | | `X` is a good letter -5 | | 1 -6 | | 2 -7 | | 3 -... | -15 | | X2 Y2 Z2 -16 | | X3 Y3 Z3 - | |___________- `Y` is a good letter too - -"#); -} - -#[test] -fn long_snippet_multiple_spans() { - test_harness(r#" -fn foo() { - X0 Y0 Z0 -1 -2 -3 - X1 Y1 Z1 -4 -5 -6 - X2 Y2 Z2 -7 -8 -9 -10 - X3 Y3 Z3 -} -"#, - vec![ - SpanLabel { - start: Position { - string: "Y0", - count: 1, - }, - end: Position { - string: "Y3", - count: 1, - }, - label: "`Y` is a good letter", - }, - SpanLabel { - start: Position { - string: "Z1", - count: 1, - }, - end: Position { - string: "Z2", - count: 1, - }, - label: "`Z` is a good letter too", - }, - ], - r#" -error: foo - --> test.rs:3:6 - | -3 | X0 Y0 Z0 - | ______^ -4 | | 1 -5 | | 2 -6 | | 3 -7 | | X1 Y1 Z1 - | |_________- -8 | || 4 -9 | || 5 -10 | || 6 -11 | || X2 Y2 Z2 - | ||__________- `Z` is a good letter too -... | -15 | | 10 -16 | | X3 Y3 Z3 - | |_______^ `Y` is a good letter - -"#); -} diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 6e1bb85ce1a..4d08d0974c1 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -23,9 +23,6 @@ use smallvec::{SmallVec, smallvec}; use std::{iter, mem}; -#[cfg(test)] -mod tests; - /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -218,7 +215,7 @@ impl TokenStream { self.0.len() } - pub(crate) fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { + pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { match streams.len() { 0 => TokenStream::default(), 1 => streams.pop().unwrap(), diff --git a/src/libsyntax/tokenstream/tests.rs b/src/libsyntax/tokenstream/tests.rs deleted file mode 100644 index 5017e5f5424..00000000000 --- a/src/libsyntax/tokenstream/tests.rs +++ /dev/null @@ -1,108 +0,0 @@ -use super::*; - -use crate::ast::Name; -use crate::with_default_globals; -use crate::tests::string_to_stream; -use syntax_pos::{Span, BytePos}; - -fn string_to_ts(string: &str) -> TokenStream { - string_to_stream(string.to_owned()) -} - -fn sp(a: u32, b: u32) -> Span { - Span::with_root_ctxt(BytePos(a), BytePos(b)) -} - -#[test] -fn test_concat() { - with_default_globals(|| { - let test_res = string_to_ts("foo::bar::baz"); - let test_fst = string_to_ts("foo::bar"); - let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); - assert_eq!(test_res.trees().count(), 5); - assert_eq!(eq_res.trees().count(), 5); - assert_eq!(test_res.eq_unspanned(&eq_res), true); - }) -} - -#[test] -fn test_to_from_bijection() { - with_default_globals(|| { - let test_start = string_to_ts("foo::bar(baz)"); - let test_end = test_start.trees().collect(); - assert_eq!(test_start, test_end) - }) -} - -#[test] -fn test_eq_0() { - with_default_globals(|| { - let test_res = string_to_ts("foo"); - let test_eqs = string_to_ts("foo"); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_eq_1() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("::bar::baz"); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_eq_3() { - with_default_globals(|| { - let test_res = string_to_ts(""); - let test_eqs = string_to_ts(""); - assert_eq!(test_res, test_eqs) - }) -} - -#[test] -fn test_diseq_0() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("bar::baz"); - assert_eq!(test_res == test_eqs, false) - }) -} - -#[test] -fn test_diseq_1() { - with_default_globals(|| { - let test_res = string_to_ts("(bar,baz)"); - let test_eqs = string_to_ts("bar,baz"); - assert_eq!(test_res == test_eqs, false) - }) -} - -#[test] -fn test_is_empty() { - with_default_globals(|| { - let test0: TokenStream = Vec::::new().into_iter().collect(); - let test1: TokenStream = - TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); - let test2 = string_to_ts("foo(bar::baz)"); - - assert_eq!(test0.is_empty(), true); - assert_eq!(test1.is_empty(), false); - assert_eq!(test2.is_empty(), false); - }) -} - -#[test] -fn test_dotdotdot() { - with_default_globals(|| { - let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); - builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); - let stream = builder.build(); - assert!(stream.eq_unspanned(&string_to_ts("..."))); - assert_eq!(stream.trees().count(), 1); - }) -} diff --git a/src/libsyntax/util/comments.rs b/src/libsyntax/util/comments.rs index 448b4f3b825..ef9226a8879 100644 --- a/src/libsyntax/util/comments.rs +++ b/src/libsyntax/util/comments.rs @@ -47,7 +47,8 @@ crate fn is_block_doc_comment(s: &str) -> bool { res } -crate fn is_doc_comment(s: &str) -> bool { +// FIXME(#64197): Try to privatize this again. +pub fn is_doc_comment(s: &str) -> bool { (s.starts_with("///") && is_line_doc_comment(s)) || s.starts_with("//!") || (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!") } diff --git a/src/libsyntax_expand/config.rs b/src/libsyntax_expand/config.rs new file mode 100644 index 00000000000..9acb9948926 --- /dev/null +++ b/src/libsyntax_expand/config.rs @@ -0,0 +1,365 @@ +use syntax::attr::HasAttrs; +use syntax::feature_gate::{ + feature_err, + EXPLAIN_STMT_ATTR_SYNTAX, + Features, + get_features, + GateIssue, +}; +use syntax::attr; +use syntax::ast; +use syntax::edition::Edition; +use syntax::mut_visit::*; +use syntax::parse::{self, validate_attr}; +use syntax::ptr::P; +use syntax::sess::ParseSess; +use syntax::symbol::sym; +use syntax::util::map_in_place::MapInPlace; + +use errors::Applicability; +use smallvec::SmallVec; + +/// A folder that strips out items that do not belong in the current configuration. +pub struct StripUnconfigured<'a> { + pub sess: &'a ParseSess, + pub features: Option<&'a Features>, +} + +// `cfg_attr`-process the crate's attributes and compute the crate's features. +pub fn features(mut krate: ast::Crate, sess: &ParseSess, edition: Edition, + allow_features: &Option>) -> (ast::Crate, Features) { + let features; + { + let mut strip_unconfigured = StripUnconfigured { + sess, + features: None, + }; + + let unconfigured_attrs = krate.attrs.clone(); + let err_count = sess.span_diagnostic.err_count(); + if let Some(attrs) = strip_unconfigured.configure(krate.attrs) { + krate.attrs = attrs; + } else { // the entire crate is unconfigured + krate.attrs = Vec::new(); + krate.module.items = Vec::new(); + return (krate, Features::new()); + } + + features = get_features(&sess.span_diagnostic, &krate.attrs, edition, allow_features); + + // Avoid reconfiguring malformed `cfg_attr`s + if err_count == sess.span_diagnostic.err_count() { + strip_unconfigured.features = Some(&features); + strip_unconfigured.configure(unconfigured_attrs); + } + } + + (krate, features) +} + +#[macro_export] +macro_rules! configure { + ($this:ident, $node:ident) => { + match $this.configure($node) { + Some(node) => node, + None => return Default::default(), + } + } +} + +impl<'a> StripUnconfigured<'a> { + pub fn configure(&mut self, mut node: T) -> Option { + self.process_cfg_attrs(&mut node); + if self.in_cfg(node.attrs()) { Some(node) } else { None } + } + + /// Parse and expand all `cfg_attr` attributes into a list of attributes + /// that are within each `cfg_attr` that has a true configuration predicate. + /// + /// Gives compiler warnigns if any `cfg_attr` does not contain any + /// attributes and is in the original source code. Gives compiler errors if + /// the syntax of any `cfg_attr` is incorrect. + pub fn process_cfg_attrs(&mut self, node: &mut T) { + node.visit_attrs(|attrs| { + attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr)); + }); + } + + /// Parse and expand a single `cfg_attr` attribute into a list of attributes + /// when the configuration predicate is true, or otherwise expand into an + /// empty list of attributes. + /// + /// Gives a compiler warning when the `cfg_attr` contains no attributes and + /// is in the original source file. Gives a compiler error if the syntax of + /// the attribute is incorrect. + fn process_cfg_attr(&mut self, attr: ast::Attribute) -> Vec { + if !attr.has_name(sym::cfg_attr) { + return vec![attr]; + } + if attr.get_normal_item().tokens.is_empty() { + self.sess.span_diagnostic + .struct_span_err( + attr.span, + "malformed `cfg_attr` attribute input", + ).span_suggestion( + attr.span, + "missing condition and attribute", + "#[cfg_attr(condition, attribute, other_attribute, ...)]".to_owned(), + Applicability::HasPlaceholders, + ).note("for more information, visit \ + ") + .emit(); + return vec![]; + } + + let res = parse::parse_in_attr(self.sess, &attr, |p| p.parse_cfg_attr()); + let (cfg_predicate, expanded_attrs) = match res { + Ok(result) => result, + Err(mut e) => { + e.emit(); + return vec![]; + } + }; + + // Lint on zero attributes in source. + if expanded_attrs.is_empty() { + return vec![attr]; + } + + // At this point we know the attribute is considered used. + attr::mark_used(&attr); + + if attr::cfg_matches(&cfg_predicate, self.sess, self.features) { + // We call `process_cfg_attr` recursively in case there's a + // `cfg_attr` inside of another `cfg_attr`. E.g. + // `#[cfg_attr(false, cfg_attr(true, some_attr))]`. + expanded_attrs.into_iter() + .flat_map(|(item, span)| self.process_cfg_attr(attr::mk_attr_from_item( + attr.style, + item, + span, + ))) + .collect() + } else { + vec![] + } + } + + /// Determines if a node with the given attributes should be included in this configuration. + pub fn in_cfg(&self, attrs: &[ast::Attribute]) -> bool { + attrs.iter().all(|attr| { + if !is_cfg(attr) { + return true; + } + + let error = |span, msg, suggestion: &str| { + let mut err = self.sess.span_diagnostic.struct_span_err(span, msg); + if !suggestion.is_empty() { + err.span_suggestion( + span, + "expected syntax is", + suggestion.into(), + Applicability::MaybeIncorrect, + ); + } + err.emit(); + true + }; + + let meta_item = match validate_attr::parse_meta(self.sess, attr) { + Ok(meta_item) => meta_item, + Err(mut err) => { err.emit(); return true; } + }; + let nested_meta_items = if let Some(nested_meta_items) = meta_item.meta_item_list() { + nested_meta_items + } else { + return error(meta_item.span, "`cfg` is not followed by parentheses", + "cfg(/* predicate */)"); + }; + + if nested_meta_items.is_empty() { + return error(meta_item.span, "`cfg` predicate is not specified", ""); + } else if nested_meta_items.len() > 1 { + return error(nested_meta_items.last().unwrap().span(), + "multiple `cfg` predicates are specified", ""); + } + + match nested_meta_items[0].meta_item() { + Some(meta_item) => attr::cfg_matches(meta_item, self.sess, self.features), + None => error(nested_meta_items[0].span(), + "`cfg` predicate key cannot be a literal", ""), + } + }) + } + + /// Visit attributes on expression and statements (but not attributes on items in blocks). + fn visit_expr_attrs(&mut self, attrs: &[ast::Attribute]) { + // flag the offending attributes + for attr in attrs.iter() { + self.maybe_emit_expr_attr_err(attr); + } + } + + /// If attributes are not allowed on expressions, emit an error for `attr` + pub fn maybe_emit_expr_attr_err(&self, attr: &ast::Attribute) { + if !self.features.map(|features| features.stmt_expr_attributes).unwrap_or(true) { + let mut err = feature_err(self.sess, + sym::stmt_expr_attributes, + attr.span, + GateIssue::Language, + EXPLAIN_STMT_ATTR_SYNTAX); + + if attr.is_doc_comment() { + err.help("`///` is for documentation comments. For a plain comment, use `//`."); + } + + err.emit(); + } + } + + pub fn configure_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + let ast::ForeignMod { abi: _, items } = foreign_mod; + items.flat_map_in_place(|item| self.configure(item)); + } + + pub fn configure_generic_params(&mut self, params: &mut Vec) { + params.flat_map_in_place(|param| self.configure(param)); + } + + fn configure_variant_data(&mut self, vdata: &mut ast::VariantData) { + match vdata { + ast::VariantData::Struct(fields, ..) | ast::VariantData::Tuple(fields, _) => + fields.flat_map_in_place(|field| self.configure(field)), + ast::VariantData::Unit(_) => {} + } + } + + pub fn configure_item_kind(&mut self, item: &mut ast::ItemKind) { + match item { + ast::ItemKind::Struct(def, _generics) | + ast::ItemKind::Union(def, _generics) => self.configure_variant_data(def), + ast::ItemKind::Enum(ast::EnumDef { variants }, _generics) => { + variants.flat_map_in_place(|variant| self.configure(variant)); + for variant in variants { + self.configure_variant_data(&mut variant.data); + } + } + _ => {} + } + } + + pub fn configure_expr_kind(&mut self, expr_kind: &mut ast::ExprKind) { + match expr_kind { + ast::ExprKind::Match(_m, arms) => { + arms.flat_map_in_place(|arm| self.configure(arm)); + } + ast::ExprKind::Struct(_path, fields, _base) => { + fields.flat_map_in_place(|field| self.configure(field)); + } + _ => {} + } + } + + pub fn configure_expr(&mut self, expr: &mut P) { + self.visit_expr_attrs(expr.attrs()); + + // If an expr is valid to cfg away it will have been removed by the + // outer stmt or expression folder before descending in here. + // Anything else is always required, and thus has to error out + // in case of a cfg attr. + // + // N.B., this is intentionally not part of the visit_expr() function + // in order for filter_map_expr() to be able to avoid this check + if let Some(attr) = expr.attrs().iter().find(|a| is_cfg(a)) { + let msg = "removing an expression is not supported in this position"; + self.sess.span_diagnostic.span_err(attr.span, msg); + } + + self.process_cfg_attrs(expr) + } + + pub fn configure_pat(&mut self, pat: &mut P) { + if let ast::PatKind::Struct(_path, fields, _etc) = &mut pat.kind { + fields.flat_map_in_place(|field| self.configure(field)); + } + } + + pub fn configure_fn_decl(&mut self, fn_decl: &mut ast::FnDecl) { + fn_decl.inputs.flat_map_in_place(|arg| self.configure(arg)); + } +} + +impl<'a> MutVisitor for StripUnconfigured<'a> { + fn visit_foreign_mod(&mut self, foreign_mod: &mut ast::ForeignMod) { + self.configure_foreign_mod(foreign_mod); + noop_visit_foreign_mod(foreign_mod, self); + } + + fn visit_item_kind(&mut self, item: &mut ast::ItemKind) { + self.configure_item_kind(item); + noop_visit_item_kind(item, self); + } + + fn visit_expr(&mut self, expr: &mut P) { + self.configure_expr(expr); + self.configure_expr_kind(&mut expr.kind); + noop_visit_expr(expr, self); + } + + fn filter_map_expr(&mut self, expr: P) -> Option> { + let mut expr = configure!(self, expr); + self.configure_expr_kind(&mut expr.kind); + noop_visit_expr(&mut expr, self); + Some(expr) + } + + fn flat_map_stmt(&mut self, stmt: ast::Stmt) -> SmallVec<[ast::Stmt; 1]> { + noop_flat_map_stmt(configure!(self, stmt), self) + } + + fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + noop_flat_map_item(configure!(self, item), self) + } + + fn flat_map_impl_item(&mut self, item: ast::ImplItem) -> SmallVec<[ast::ImplItem; 1]> { + noop_flat_map_impl_item(configure!(self, item), self) + } + + fn flat_map_trait_item(&mut self, item: ast::TraitItem) -> SmallVec<[ast::TraitItem; 1]> { + noop_flat_map_trait_item(configure!(self, item), self) + } + + fn visit_mac(&mut self, _mac: &mut ast::Mac) { + // Don't configure interpolated AST (cf. issue #34171). + // Interpolated AST will get configured once the surrounding tokens are parsed. + } + + fn visit_pat(&mut self, pat: &mut P) { + self.configure_pat(pat); + noop_visit_pat(pat, self) + } + + fn visit_fn_decl(&mut self, mut fn_decl: &mut P) { + self.configure_fn_decl(&mut fn_decl); + noop_visit_fn_decl(fn_decl, self); + } +} + +fn is_cfg(attr: &ast::Attribute) -> bool { + attr.check_name(sym::cfg) +} + +/// Process the potential `cfg` attributes on a module. +/// Also determine if the module should be included in this configuration. +pub fn process_configure_mod( + sess: &ParseSess, + cfg_mods: bool, + attrs: &[ast::Attribute], +) -> (bool, Vec) { + // Don't perform gated feature checking. + let mut strip_unconfigured = StripUnconfigured { sess, features: None }; + let mut attrs = attrs.to_owned(); + strip_unconfigured.process_cfg_attrs(&mut attrs); + (!cfg_mods || strip_unconfigured.in_cfg(&attrs), attrs) +} diff --git a/src/libsyntax_expand/expand.rs b/src/libsyntax_expand/expand.rs index d0b60fa308d..1e6c23087d8 100644 --- a/src/libsyntax_expand/expand.rs +++ b/src/libsyntax_expand/expand.rs @@ -3,13 +3,13 @@ use crate::proc_macro::{collect_derives, MarkAttrs}; use crate::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind}; use crate::mbe::macro_rules::annotate_err_with_kind; use crate::placeholders::{placeholder, PlaceholderExpander}; +use crate::config::StripUnconfigured; +use crate::configure; use syntax::ast::{self, AttrItem, Block, Ident, LitKind, NodeId, PatKind, Path}; use syntax::ast::{MacStmtStyle, StmtKind, ItemKind}; use syntax::attr::{self, HasAttrs}; use syntax::source_map::respan; -use syntax::configure; -use syntax::config::StripUnconfigured; use syntax::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err}; use syntax::mut_visit::*; use syntax::parse::DirectoryOwnership; diff --git a/src/libsyntax_expand/lib.rs b/src/libsyntax_expand/lib.rs index 10eb3ecb20b..46d59dd249c 100644 --- a/src/libsyntax_expand/lib.rs +++ b/src/libsyntax_expand/lib.rs @@ -33,6 +33,35 @@ pub use mbe::macro_rules::compile_declarative_macro; pub mod base; pub mod build; pub mod expand; +#[macro_use] pub mod config; pub mod proc_macro; crate mod mbe; + +// HACK(Centril, #64197): These shouldn't really be here. +// Rather, they should be with their respective modules which are defined in other crates. +// However, since for now constructing a `ParseSess` sorta requires `config` from this crate, +// these tests will need to live here in the iterim. + +#[cfg(test)] +mod tests; +#[cfg(test)] +mod parse { + #[cfg(test)] + mod tests; + #[cfg(test)] + mod lexer { + #[cfg(test)] + mod tests; + } +} +#[cfg(test)] +mod tokenstream { + #[cfg(test)] + mod tests; +} +#[cfg(test)] +mod mut_visit { + #[cfg(test)] + mod tests; +} diff --git a/src/libsyntax_expand/mut_visit/tests.rs b/src/libsyntax_expand/mut_visit/tests.rs new file mode 100644 index 00000000000..30e812a1179 --- /dev/null +++ b/src/libsyntax_expand/mut_visit/tests.rs @@ -0,0 +1,70 @@ +use crate::tests::{string_to_crate, matches_codepattern}; + +use syntax::ast::{self, Ident}; +use syntax::print::pprust; +use syntax::mut_visit::{self, MutVisitor}; +use syntax::with_default_globals; + +// This version doesn't care about getting comments or doc-strings in. +fn fake_print_crate(s: &mut pprust::State<'_>, + krate: &ast::Crate) { + s.print_mod(&krate.module, &krate.attrs) +} + +// Change every identifier to "zz". +struct ToZzIdentMutVisitor; + +impl MutVisitor for ToZzIdentMutVisitor { + fn visit_ident(&mut self, ident: &mut ast::Ident) { + *ident = Ident::from_str("zz"); + } + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +// Maybe add to `expand.rs`. +macro_rules! assert_pred { + ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( + { + let pred_val = $pred; + let a_val = $a; + let b_val = $b; + if !(pred_val(&a_val, &b_val)) { + panic!("expected args satisfying {}, got {} and {}", + $predname, a_val, b_val); + } + } + ) +} + +// Make sure idents get transformed everywhere. +#[test] fn ident_transformation () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); + }) +} + +// Make sure idents get transformed even inside macro defs. +#[test] fn ident_transformation_in_defs () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ + (g $(d $d $e)+))} ".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string()); + }) +} diff --git a/src/libsyntax_expand/parse/lexer/tests.rs b/src/libsyntax_expand/parse/lexer/tests.rs new file mode 100644 index 00000000000..3d5726fa601 --- /dev/null +++ b/src/libsyntax_expand/parse/lexer/tests.rs @@ -0,0 +1,277 @@ +use crate::config::process_configure_mod; + +use rustc_data_structures::sync::Lrc; +use syntax::token::{self, Token, TokenKind}; +use syntax::sess::ParseSess; +use syntax::source_map::{SourceMap, FilePathMapping}; +use syntax::util::comments::is_doc_comment; +use syntax::with_default_globals; +use syntax::parse::lexer::StringReader; +use syntax_pos::symbol::Symbol; + +use errors::{Handler, emitter::EmitterWriter}; +use std::io; +use std::path::PathBuf; +use syntax_pos::{BytePos, Span}; + +fn mk_sess(sm: Lrc) -> ParseSess { + let emitter = EmitterWriter::new( + Box::new(io::sink()), + Some(sm.clone()), + false, + false, + false, + None, + false, + ); + ParseSess::with_span_handler( + Handler::with_emitter(true, None, Box::new(emitter)), + sm, + process_configure_mod, + ) +} + +// Creates a string reader for the given string. +fn setup<'a>(sm: &SourceMap, + sess: &'a ParseSess, + teststr: String) + -> StringReader<'a> { + let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); + StringReader::new(sess, sf, None) +} + +#[test] +fn t1() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut string_reader = setup( + &sm, + &sh, + "/* my source file */ fn main() { println!(\"zebra\"); }\n".to_string(), + ); + assert_eq!(string_reader.next_token(), token::Comment); + assert_eq!(string_reader.next_token(), token::Whitespace); + let tok1 = string_reader.next_token(); + let tok2 = Token::new( + mk_ident("fn"), + Span::with_root_ctxt(BytePos(21), BytePos(23)), + ); + assert_eq!(tok1.kind, tok2.kind); + assert_eq!(tok1.span, tok2.span); + assert_eq!(string_reader.next_token(), token::Whitespace); + // Read another token. + let tok3 = string_reader.next_token(); + assert_eq!(string_reader.pos.clone(), BytePos(28)); + let tok4 = Token::new( + mk_ident("main"), + Span::with_root_ctxt(BytePos(24), BytePos(28)), + ); + assert_eq!(tok3.kind, tok4.kind); + assert_eq!(tok3.span, tok4.span); + + assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); + assert_eq!(string_reader.pos.clone(), BytePos(29)) + }) +} + +// Checks that the given reader produces the desired stream +// of tokens (stop checking after exhausting `expected`). +fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec) { + for expected_tok in &expected { + assert_eq!(&string_reader.next_token(), expected_tok); + } +} + +// Makes the identifier by looking up the string in the interner. +fn mk_ident(id: &str) -> TokenKind { + token::Ident(Symbol::intern(id), false) +} + +fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { + TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) +} + +#[test] +fn doublecolon_parsing() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a b".to_string()), + vec![mk_ident("a"), token::Whitespace, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_2() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a::b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_3() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a ::b".to_string()), + vec![mk_ident("a"), token::Whitespace, token::Colon, token::Colon, mk_ident("b")], + ); + }) +} + +#[test] +fn doublecolon_parsing_4() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization( + setup(&sm, &sh, "a:: b".to_string()), + vec![mk_ident("a"), token::Colon, token::Colon, token::Whitespace, mk_ident("b")], + ); + }) +} + +#[test] +fn character_a() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "'a'".to_string()).next_token(), + mk_lit(token::Char, "a", None), + ); + }) +} + +#[test] +fn character_space() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "' '".to_string()).next_token(), + mk_lit(token::Char, " ", None), + ); + }) +} + +#[test] +fn character_escaped() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "'\\n'".to_string()).next_token(), + mk_lit(token::Char, "\\n", None), + ); + }) +} + +#[test] +fn lifetime_name() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "'abc".to_string()).next_token(), + token::Lifetime(Symbol::intern("'abc")), + ); + }) +} + +#[test] +fn raw_string() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!( + setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), + mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None), + ); + }) +} + +#[test] +fn literal_suffixes() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + macro_rules! test { + ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ + assert_eq!( + setup(&sm, &sh, format!("{}suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, Some("suffix")), + ); + // with a whitespace separator + assert_eq!( + setup(&sm, &sh, format!("{} suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, None), + ); + }} + } + + test!("'a'", Char, "a"); + test!("b'a'", Byte, "a"); + test!("\"a\"", Str, "a"); + test!("b\"a\"", ByteStr, "a"); + test!("1234", Integer, "1234"); + test!("0b101", Integer, "0b101"); + test!("0xABC", Integer, "0xABC"); + test!("1.0", Float, "1.0"); + test!("1.0e10", Float, "1.0e10"); + + assert_eq!( + setup(&sm, &sh, "2us".to_string()).next_token(), + mk_lit(token::Integer, "2", Some("us")), + ); + assert_eq!( + setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::StrRaw(3), "raw", Some("suffix")), + ); + assert_eq!( + setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")), + ); + }) +} + +#[test] +fn line_doc_comments() { + assert!(is_doc_comment("///")); + assert!(is_doc_comment("/// blah")); + assert!(!is_doc_comment("////")); +} + +#[test] +fn nested_block_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); + assert_eq!(lexer.next_token(), token::Comment); + assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn crlf_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); + let comment = lexer.next_token(); + assert_eq!(comment.kind, token::Comment); + assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); + assert_eq!(lexer.next_token(), token::Whitespace); + assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); + }) +} diff --git a/src/libsyntax_expand/parse/tests.rs b/src/libsyntax_expand/parse/tests.rs new file mode 100644 index 00000000000..01739809f7f --- /dev/null +++ b/src/libsyntax_expand/parse/tests.rs @@ -0,0 +1,338 @@ +use crate::config::process_configure_mod; +use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; + +use syntax::ast::{self, Name, PatKind}; +use syntax::attr::first_attr_value_str_by_name; +use syntax::sess::ParseSess; +use syntax::token::{self, Token}; +use syntax::print::pprust::item_to_string; +use syntax::ptr::P; +use syntax::source_map::FilePathMapping; +use syntax::symbol::{kw, sym}; +use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream}; +use syntax::visit; +use syntax::with_default_globals; +use syntax::parse::new_parser_from_source_str; +use syntax_pos::{Span, BytePos, Pos, FileName}; +use errors::PResult; + +use std::path::PathBuf; + +fn sess() -> ParseSess { + ParseSess::new(FilePathMapping::empty(), process_configure_mod) +} + +/// Parses an item. +/// +/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` +/// when a syntax error occurred. +fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) + -> PResult<'_, Option>> { + new_parser_from_source_str(sess, name, source).parse_item() +} + +// Produces a `syntax_pos::span`. +fn sp(a: u32, b: u32) -> Span { + Span::with_root_ctxt(BytePos(a), BytePos(b)) +} + +/// Parses a string, return an expression. +fn string_to_expr(source_str : String) -> P { + with_error_checking_parse(source_str, &sess(), |p| p.parse_expr()) +} + +/// Parses a string, returns an item. +fn string_to_item(source_str : String) -> Option> { + with_error_checking_parse(source_str, &sess(), |p| p.parse_item()) +} + +#[should_panic] +#[test] fn bad_path_expr_1() { + with_default_globals(|| { + string_to_expr("::abc::def::return".to_string()); + }) +} + +// Checks the token-tree-ization of macros. +#[test] +fn string_to_tts_macro () { + with_default_globals(|| { + let tts: Vec<_> = + string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); + let tts: &[TokenTree] = &tts[..]; + + match tts { + [ + TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), + TokenTree::Token(Token { kind: token::Not, .. }), + TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), + TokenTree::Delimited(_, macro_delim, macro_tts) + ] + if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => { + let tts = ¯o_tts.trees().collect::>(); + match &tts[..] { + [ + TokenTree::Delimited(_, first_delim, first_tts), + TokenTree::Token(Token { kind: token::FatArrow, .. }), + TokenTree::Delimited(_, second_delim, second_tts), + ] + if macro_delim == &token::Paren => { + let tts = &first_tts.trees().collect::>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if first_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), + } + let tts = &second_tts.trees().collect::>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if second_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), + } + }, + _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), + } + }, + _ => panic!("value: {:?}",tts), + } + }) +} + +#[test] +fn string_to_tts_1() { + with_default_globals(|| { + let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); + + let expected = TokenStream::new(vec![ + TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), + TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(5, 6), sp(13, 14)), + token::DelimToken::Paren, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), + TokenTree::token(token::Colon, sp(8, 9)).into(), + TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), + ]).into(), + ).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(15, 16), sp(20, 21)), + token::DelimToken::Brace, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), + TokenTree::token(token::Semi, sp(18, 19)).into(), + ]).into(), + ).into() + ]); + + assert_eq!(tts, expected); + }) +} + +#[test] fn parse_use() { + with_default_globals(|| { + let use_s = "use foo::bar::baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + + let use_s = "use foo::bar as baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + }) +} + +#[test] fn parse_extern_crate() { + with_default_globals(|| { + let ex_s = "extern crate foo;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + + let ex_s = "extern crate foo as bar;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + }) +} + +fn get_spans_of_pat_idents(src: &str) -> Vec { + let item = string_to_item(src.to_string()).unwrap(); + + struct PatIdentVisitor { + spans: Vec + } + impl<'a> visit::Visitor<'a> for PatIdentVisitor { + fn visit_pat(&mut self, p: &'a ast::Pat) { + match p.kind { + PatKind::Ident(_ , ref ident, _) => { + self.spans.push(ident.span.clone()); + } + _ => { + visit::walk_pat(self, p); + } + } + } + } + let mut v = PatIdentVisitor { spans: Vec::new() }; + visit::walk_item(&mut v, &item); + return v.spans; +} + +#[test] fn span_of_self_arg_pat_idents_are_correct() { + with_default_globals(|| { + + let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", + "impl z { fn a (&mut self, &myarg: i32) {} }", + "impl z { fn a (&'a self, &myarg: i32) {} }", + "impl z { fn a (self, &myarg: i32) {} }", + "impl z { fn a (self: Foo, &myarg: i32) {} }", + ]; + + for &src in &srcs { + let spans = get_spans_of_pat_idents(src); + let (lo, hi) = (spans[0].lo(), spans[0].hi()); + assert!("self" == &src[lo.to_usize()..hi.to_usize()], + "\"{}\" != \"self\". src=\"{}\"", + &src[lo.to_usize()..hi.to_usize()], src) + } + }) +} + +#[test] fn parse_exprs () { + with_default_globals(|| { + // just make sure that they parse.... + string_to_expr("3 + 4".to_string()); + string_to_expr("a::z.froob(b,&(987+3))".to_string()); + }) +} + +#[test] fn attrs_fix_bug () { + with_default_globals(|| { + string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) + -> Result, String> { +#[cfg(windows)] +fn wb() -> c_int { + (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int +} + +#[cfg(unix)] +fn wb() -> c_int { O_WRONLY as c_int } + +let mut fflags: c_int = wb(); +}".to_string()); + }) +} + +#[test] fn crlf_doc_comments() { + with_default_globals(|| { + let sess = sess(); + + let name_1 = FileName::Custom("crlf_source_1".to_string()); + let source = "/// doc comment\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_1, source, &sess) + .unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/// doc comment"); + + let name_2 = FileName::Custom("crlf_source_2".to_string()); + let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_2, source, &sess) + .unwrap().unwrap(); + let docs = item.attrs.iter().filter(|a| a.has_name(sym::doc)) + .map(|a| a.value_str().unwrap().to_string()).collect::>(); + let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; + assert_eq!(&docs[..], b); + + let name_3 = FileName::Custom("clrf_source_3".to_string()); + let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); + }); +} + +#[test] +fn ttdelim_span() { + fn parse_expr_from_source_str( + name: FileName, source: String, sess: &ParseSess + ) -> PResult<'_, P> { + new_parser_from_source_str(sess, name, source).parse_expr() + } + + with_default_globals(|| { + let sess = sess(); + let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), + "foo!( fn main() { body } )".to_string(), &sess).unwrap(); + + let tts: Vec<_> = match expr.kind { + ast::ExprKind::Mac(ref mac) => mac.stream().trees().collect(), + _ => panic!("not a macro"), + }; + + let span = tts.iter().rev().next().unwrap().span(); + + match sess.source_map().span_to_snippet(span) { + Ok(s) => assert_eq!(&s[..], "{ body }"), + Err(_) => panic!("could not get snippet"), + } + }); +} + +// This tests that when parsing a string (rather than a file) we don't try +// and read in a file for a module declaration and just parse a stub. +// See `recurse_into_file_modules` in the parser. +#[test] +fn out_of_line_mod() { + with_default_globals(|| { + let item = parse_item_from_source_str( + PathBuf::from("foo").into(), + "mod foo { struct S; mod this_does_not_exist; }".to_owned(), + &sess(), + ).unwrap().unwrap(); + + if let ast::ItemKind::Mod(ref m) = item.kind { + assert!(m.items.len() == 2); + } else { + panic!(); + } + }); +} + +#[test] +fn eqmodws() { + assert_eq!(matches_codepattern("",""),true); + assert_eq!(matches_codepattern("","a"),false); + assert_eq!(matches_codepattern("a",""),false); + assert_eq!(matches_codepattern("a","a"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); + assert_eq!(matches_codepattern("a b","a b"),true); + assert_eq!(matches_codepattern("ab","a b"),false); + assert_eq!(matches_codepattern("a b","ab"),true); + assert_eq!(matches_codepattern(" a b","ab"),true); +} + +#[test] +fn pattern_whitespace() { + assert_eq!(matches_codepattern("","\x0C"), false); + assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); +} + +#[test] +fn non_pattern_whitespace() { + // These have the property 'White_Space' but not 'Pattern_White_Space' + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); + assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); +} diff --git a/src/libsyntax_expand/tests.rs b/src/libsyntax_expand/tests.rs new file mode 100644 index 00000000000..461d9a43954 --- /dev/null +++ b/src/libsyntax_expand/tests.rs @@ -0,0 +1,1254 @@ +use crate::config::process_configure_mod; +use syntax::ast; +use syntax::tokenstream::TokenStream; +use syntax::sess::ParseSess; +use syntax::source_map::{SourceMap, FilePathMapping}; +use syntax::with_default_globals; +use syntax::parse::{source_file_to_stream, new_parser_from_source_str, parser::Parser}; +use syntax_pos::{BytePos, Span, MultiSpan}; + +use errors::emitter::EmitterWriter; +use errors::{PResult, Handler}; +use rustc_data_structures::sync::Lrc; + +use std::io; +use std::io::prelude::*; +use std::iter::Peekable; +use std::path::{Path, PathBuf}; +use std::str; +use std::sync::{Arc, Mutex}; + +/// Map string to parser (via tts). +fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { + new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) +} + +crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where + F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, +{ + let mut p = string_to_parser(&ps, s); + let x = f(&mut p).unwrap(); + p.sess.span_diagnostic.abort_if_errors(); + x +} + +/// Maps a string to tts, using a made-up filename. +crate fn string_to_stream(source_str: String) -> TokenStream { + let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod); + source_file_to_stream( + &ps, + ps.source_map().new_source_file(PathBuf::from("bogofile").into(), + source_str, + ), None).0 +} + +/// Parses a string, returns a crate. +crate fn string_to_crate(source_str : String) -> ast::Crate { + let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_crate_mod() + }) +} + +/// Does the given string match the pattern? whitespace in the first string +/// may be deleted or replaced with other whitespace to match the pattern. +/// This function is relatively Unicode-ignorant; fortunately, the careful design +/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). +crate fn matches_codepattern(a : &str, b : &str) -> bool { + let mut a_iter = a.chars().peekable(); + let mut b_iter = b.chars().peekable(); + + loop { + let (a, b) = match (a_iter.peek(), b_iter.peek()) { + (None, None) => return true, + (None, _) => return false, + (Some(&a), None) => { + if rustc_lexer::is_whitespace(a) { + break // Trailing whitespace check is out of loop for borrowck. + } else { + return false + } + } + (Some(&a), Some(&b)) => (a, b) + }; + + if rustc_lexer::is_whitespace(a) && rustc_lexer::is_whitespace(b) { + // Skip whitespace for `a` and `b`. + scan_for_non_ws_or_end(&mut a_iter); + scan_for_non_ws_or_end(&mut b_iter); + } else if rustc_lexer::is_whitespace(a) { + // Skip whitespace for `a`. + scan_for_non_ws_or_end(&mut a_iter); + } else if a == b { + a_iter.next(); + b_iter.next(); + } else { + return false + } + } + + // Check if a has *only* trailing whitespace. + a_iter.all(rustc_lexer::is_whitespace) +} + +/// Advances the given peekable `Iterator` until it reaches a non-whitespace character. +fn scan_for_non_ws_or_end>(iter: &mut Peekable) { + while iter.peek().copied().map(|c| rustc_lexer::is_whitespace(c)) == Some(true) { + iter.next(); + } +} + +/// Identifies a position in the text by the n'th occurrence of a string. +struct Position { + string: &'static str, + count: usize, +} + +struct SpanLabel { + start: Position, + end: Position, + label: &'static str, +} + +crate struct Shared { + pub data: Arc>, +} + +impl Write for Shared { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.data.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.data.lock().unwrap().flush() + } +} + +fn test_harness(file_text: &str, span_labels: Vec, expected_output: &str) { + with_default_globals(|| { + let output = Arc::new(Mutex::new(Vec::new())); + + let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); + source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned()); + + let primary_span = make_span(&file_text, &span_labels[0].start, &span_labels[0].end); + let mut msp = MultiSpan::from_span(primary_span); + for span_label in span_labels { + let span = make_span(&file_text, &span_label.start, &span_label.end); + msp.push_span_label(span, span_label.label.to_string()); + println!("span: {:?} label: {:?}", span, span_label.label); + println!("text: {:?}", source_map.span_to_snippet(span)); + } + + let emitter = EmitterWriter::new( + Box::new(Shared { data: output.clone() }), + Some(source_map.clone()), + false, + false, + false, + None, + false, + ); + let handler = Handler::with_emitter(true, None, Box::new(emitter)); + handler.span_err(msp, "foo"); + + assert!(expected_output.chars().next() == Some('\n'), + "expected output should begin with newline"); + let expected_output = &expected_output[1..]; + + let bytes = output.lock().unwrap(); + let actual_output = str::from_utf8(&bytes).unwrap(); + println!("expected output:\n------\n{}------", expected_output); + println!("actual output:\n------\n{}------", actual_output); + + assert!(expected_output == actual_output) + }) +} + +fn make_span(file_text: &str, start: &Position, end: &Position) -> Span { + let start = make_pos(file_text, start); + let end = make_pos(file_text, end) + end.string.len(); // just after matching thing ends + assert!(start <= end); + Span::with_root_ctxt(BytePos(start as u32), BytePos(end as u32)) +} + +fn make_pos(file_text: &str, pos: &Position) -> usize { + let mut remainder = file_text; + let mut offset = 0; + for _ in 0..pos.count { + if let Some(n) = remainder.find(&pos.string) { + offset += n; + remainder = &remainder[n + 1..]; + } else { + panic!("failed to find {} instances of {:?} in {:?}", + pos.count, + pos.string, + file_text); + } + } + offset +} + +#[test] +fn ends_on_col0() { + test_harness(r#" +fn foo() { +} +"#, + vec![ + SpanLabel { + start: Position { + string: "{", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "test", + }, + ], + r#" +error: foo + --> test.rs:2:10 + | +2 | fn foo() { + | __________^ +3 | | } + | |_^ test + +"#); +} + +#[test] +fn ends_on_col2() { + test_harness(r#" +fn foo() { + + + } +"#, + vec![ + SpanLabel { + start: Position { + string: "{", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "test", + }, + ], + r#" +error: foo + --> test.rs:2:10 + | +2 | fn foo() { + | __________^ +3 | | +4 | | +5 | | } + | |___^ test + +"#); +} +#[test] +fn non_nested() { + test_harness(r#" +fn foo() { + X0 Y0 + X1 Y1 + X2 Y2 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "Y2", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 + | ____^__- + | | ___| + | || +4 | || X1 Y1 +5 | || X2 Y2 + | ||____^__- `Y` is a good letter too + | |____| + | `X` is a good letter + +"#); +} + +#[test] +fn nested() { + test_harness(r#" +fn foo() { + X0 Y0 + Y1 X1 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X1", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "Y1", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], +r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 + | ____^__- + | | ___| + | || +4 | || Y1 X1 + | ||____-__^ `X` is a good letter + | |_____| + | `Y` is a good letter too + +"#); +} + +#[test] +fn different_overlap() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Z1", + count: 1, + }, + end: Position { + string: "X3", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |_________- +5 | || X2 Y2 Z2 + | ||____^ `X` is a good letter +6 | | X3 Y3 Z3 + | |_____- `Y` is a good letter too + +"#); +} + +#[test] +fn triple_overlap() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "Y2", + count: 1, + }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { + string: "Z0", + count: 1, + }, + end: Position { + string: "Z2", + count: 1, + }, + label: "`Z` label", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | X0 Y0 Z0 + | _____^__-__- + | | ____|__| + | || ___| + | ||| +4 | ||| X1 Y1 Z1 +5 | ||| X2 Y2 Z2 + | |||____^__-__- `Z` label + | ||____|__| + | |____| `Y` is a good letter too + | `X` is a good letter + +"#); +} + +#[test] +fn triple_exact_overlap() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X2", + count: 1, + }, + label: "`Z` label", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | / X0 Y0 Z0 +4 | | X1 Y1 Z1 +5 | | X2 Y2 Z2 + | | ^ + | | | + | | `X` is a good letter + | |____`Y` is a good letter too + | `Z` label + +"#); +} + +#[test] +fn minimum_depth() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "X1", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Y1", + count: 1, + }, + end: Position { + string: "Z2", + count: 1, + }, + label: "`Y` is a good letter too", + }, + SpanLabel { + start: Position { + string: "X2", + count: 1, + }, + end: Position { + string: "Y3", + count: 1, + }, + label: "`Z`", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^_- + | ||____| + | | `X` is a good letter +5 | | X2 Y2 Z2 + | |____-______- `Y` is a good letter too + | ____| + | | +6 | | X3 Y3 Z3 + | |________- `Z` + +"#); +} + +#[test] +fn non_overlaping() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "X0", + count: 1, + }, + end: Position { + string: "X1", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Y2", + count: 1, + }, + end: Position { + string: "Z3", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | / X0 Y0 Z0 +4 | | X1 Y1 Z1 + | |____^ `X` is a good letter +5 | X2 Y2 Z2 + | ______- +6 | | X3 Y3 Z3 + | |__________- `Y` is a good letter too + +"#); +} + +#[test] +fn overlaping_start_and_end() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "X1", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Z1", + count: 1, + }, + end: Position { + string: "Z3", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^____- + | ||____| + | | `X` is a good letter +5 | | X2 Y2 Z2 +6 | | X3 Y3 Z3 + | |___________- `Y` is a good letter too + +"#); +} + +#[test] +fn multiple_labels_primary_without_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { + string: "c", + count: 1, + }, + end: Position { + string: "c", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- `a` is a good letter + +"#); +} + +#[test] +fn multiple_labels_secondary_without_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ `a` is a good letter + +"#); +} + +#[test] +fn multiple_labels_primary_without_message_2() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "`b` is a good letter", + }, + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "c", + count: 1, + }, + end: Position { + string: "c", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- + | | + | `b` is a good letter + +"#); +} + +#[test] +fn multiple_labels_secondary_without_message_2() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "`b` is a good letter", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + | | + | `b` is a good letter + +"#); +} + +#[test] +fn multiple_labels_secondary_without_message_3() { + test_harness(r#" +fn foo() { + a bc d +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "b", + count: 1, + }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { + string: "c", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a bc d + | ^^^^---- + | | + | `a` is a good letter + +"#); +} + +#[test] +fn multiple_labels_without_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + +"#); +} + +#[test] +fn multiple_labels_without_message_2() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + SpanLabel { + start: Position { + string: "c", + count: 1, + }, + end: Position { + string: "c", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:7 + | +3 | a { b { c } d } + | ----^^^^-^^-- + +"#); +} + +#[test] +fn multiple_labels_with_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "`a` is a good letter", + }, + SpanLabel { + start: Position { + string: "b", + count: 1, + }, + end: Position { + string: "}", + count: 1, + }, + label: "`b` is a good letter", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^-------^^ + | | | + | | `b` is a good letter + | `a` is a good letter + +"#); +} + +#[test] +fn single_label_with_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "`a` is a good letter", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^^^^^^^^^^ `a` is a good letter + +"#); +} + +#[test] +fn single_label_without_message() { + test_harness(r#" +fn foo() { + a { b { c } d } +} +"#, + vec![ + SpanLabel { + start: Position { + string: "a", + count: 1, + }, + end: Position { + string: "d", + count: 1, + }, + label: "", + }, + ], + r#" +error: foo + --> test.rs:3:3 + | +3 | a { b { c } d } + | ^^^^^^^^^^^^^ + +"#); +} + +#[test] +fn long_snippet() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 + X1 Y1 Z1 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 + X2 Y2 Z2 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "X1", + count: 1, + }, + label: "`X` is a good letter", + }, + SpanLabel { + start: Position { + string: "Z1", + count: 1, + }, + end: Position { + string: "Z3", + count: 1, + }, + label: "`Y` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | X1 Y1 Z1 + | |____^____- + | ||____| + | | `X` is a good letter +5 | | 1 +6 | | 2 +7 | | 3 +... | +15 | | X2 Y2 Z2 +16 | | X3 Y3 Z3 + | |___________- `Y` is a good letter too + +"#); +} + +#[test] +fn long_snippet_multiple_spans() { + test_harness(r#" +fn foo() { + X0 Y0 Z0 +1 +2 +3 + X1 Y1 Z1 +4 +5 +6 + X2 Y2 Z2 +7 +8 +9 +10 + X3 Y3 Z3 +} +"#, + vec![ + SpanLabel { + start: Position { + string: "Y0", + count: 1, + }, + end: Position { + string: "Y3", + count: 1, + }, + label: "`Y` is a good letter", + }, + SpanLabel { + start: Position { + string: "Z1", + count: 1, + }, + end: Position { + string: "Z2", + count: 1, + }, + label: "`Z` is a good letter too", + }, + ], + r#" +error: foo + --> test.rs:3:6 + | +3 | X0 Y0 Z0 + | ______^ +4 | | 1 +5 | | 2 +6 | | 3 +7 | | X1 Y1 Z1 + | |_________- +8 | || 4 +9 | || 5 +10 | || 6 +11 | || X2 Y2 Z2 + | ||__________- `Z` is a good letter too +... | +15 | | 10 +16 | | X3 Y3 Z3 + | |_______^ `Y` is a good letter + +"#); +} diff --git a/src/libsyntax_expand/tokenstream/tests.rs b/src/libsyntax_expand/tokenstream/tests.rs new file mode 100644 index 00000000000..cf9fead638e --- /dev/null +++ b/src/libsyntax_expand/tokenstream/tests.rs @@ -0,0 +1,110 @@ +use crate::tests::string_to_stream; + +use syntax::ast::Name; +use syntax::token; +use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree}; +use syntax::with_default_globals; +use syntax_pos::{Span, BytePos}; +use smallvec::smallvec; + +fn string_to_ts(string: &str) -> TokenStream { + string_to_stream(string.to_owned()) +} + +fn sp(a: u32, b: u32) -> Span { + Span::with_root_ctxt(BytePos(a), BytePos(b)) +} + +#[test] +fn test_concat() { + with_default_globals(|| { + let test_res = string_to_ts("foo::bar::baz"); + let test_fst = string_to_ts("foo::bar"); + let test_snd = string_to_ts("::baz"); + let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + assert_eq!(test_res.trees().count(), 5); + assert_eq!(eq_res.trees().count(), 5); + assert_eq!(test_res.eq_unspanned(&eq_res), true); + }) +} + +#[test] +fn test_to_from_bijection() { + with_default_globals(|| { + let test_start = string_to_ts("foo::bar(baz)"); + let test_end = test_start.trees().collect(); + assert_eq!(test_start, test_end) + }) +} + +#[test] +fn test_eq_0() { + with_default_globals(|| { + let test_res = string_to_ts("foo"); + let test_eqs = string_to_ts("foo"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_1() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("::bar::baz"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_3() { + with_default_globals(|| { + let test_res = string_to_ts(""); + let test_eqs = string_to_ts(""); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_diseq_0() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("bar::baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_diseq_1() { + with_default_globals(|| { + let test_res = string_to_ts("(bar,baz)"); + let test_eqs = string_to_ts("bar,baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_is_empty() { + with_default_globals(|| { + let test0: TokenStream = Vec::::new().into_iter().collect(); + let test1: TokenStream = + TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); + let test2 = string_to_ts("foo(bar::baz)"); + + assert_eq!(test0.is_empty(), true); + assert_eq!(test1.is_empty(), false); + assert_eq!(test2.is_empty(), false); + }) +} + +#[test] +fn test_dotdotdot() { + with_default_globals(|| { + let mut builder = TokenStreamBuilder::new(); + builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); + builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); + builder.push(TokenTree::token(token::Dot, sp(2, 3))); + let stream = builder.build(); + assert!(stream.eq_unspanned(&string_to_ts("..."))); + assert_eq!(stream.trees().count(), 1); + }) +} diff --git a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs index ac864e76784..3d2181d4e37 100644 --- a/src/test/ui-fulldeps/ast_stmt_expr_attr.rs +++ b/src/test/ui-fulldeps/ast_stmt_expr_attr.rs @@ -6,6 +6,7 @@ #![feature(rustc_private)] extern crate syntax; +extern crate syntax_expand; extern crate rustc_errors; use rustc_errors::PResult; @@ -14,13 +15,13 @@ use syntax::attr::*; use syntax::ast; use syntax::sess::ParseSess; use syntax::source_map::{FilePathMapping, FileName}; -use syntax::parse; +use syntax::ptr::P; +use syntax::print::pprust; +use syntax::parse::parser::attr::*; use syntax::parse::new_parser_from_source_str; use syntax::parse::parser::Parser; use syntax::token; -use syntax::ptr::P; -use syntax::parse::parser::attr::*; -use syntax::print::pprust; +use syntax_expand::config::process_configure_mod; use std::fmt; // Copied out of syntax::util::parser_testing @@ -72,8 +73,12 @@ fn str_compare String>(e: &str, expected: &[T], actual: &[T], f: } } +fn sess() -> ParseSess { + ParseSess::new(FilePathMapping::empty(), process_configure_mod) +} + fn check_expr_attrs(es: &str, expected: &[&str]) { - let ps = ParseSess::new(FilePathMapping::empty()); + let ps = sess(); let e = expr(es, &ps).expect("parse error"); let actual = &e.attrs; str_compare(es, @@ -83,7 +88,7 @@ fn check_expr_attrs(es: &str, expected: &[&str]) { } fn check_stmt_attrs(es: &str, expected: &[&str]) { - let ps = ParseSess::new(FilePathMapping::empty()); + let ps = sess(); let e = stmt(es, &ps).expect("parse error"); let actual = e.kind.attrs(); str_compare(es, @@ -93,7 +98,7 @@ fn check_stmt_attrs(es: &str, expected: &[&str]) { } fn reject_expr_parse(es: &str) { - let ps = ParseSess::new(FilePathMapping::empty()); + let ps = sess(); match expr(es, &ps) { Ok(_) => panic!("parser did not reject `{}`", es), Err(mut e) => e.cancel(), @@ -101,7 +106,7 @@ fn reject_expr_parse(es: &str) { } fn reject_stmt_parse(es: &str) { - let ps = ParseSess::new(FilePathMapping::empty()); + let ps = sess(); match stmt(es, &ps) { Ok(_) => panic!("parser did not reject `{}`", es), Err(mut e) => e.cancel(), diff --git a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs index ac97ec70be2..472440e2e14 100644 --- a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs +++ b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs @@ -5,11 +5,13 @@ #![feature(rustc_private)] extern crate syntax; +extern crate syntax_expand; use std::path::Path; use syntax::sess::ParseSess; use syntax::source_map::FilePathMapping; -use syntax::parse; +use syntax::parse::new_parser_from_file; +use syntax_expand::config::process_configure_mod; #[path = "mod_dir_simple/test.rs"] mod gravy; @@ -21,10 +23,10 @@ pub fn main() { } fn parse() { - let parse_session = ParseSess::new(FilePathMapping::empty()); + let parse_session = ParseSess::new(FilePathMapping::empty(), process_configure_mod); let path = Path::new(file!()); let path = path.canonicalize().unwrap(); - let mut parser = parse::new_parser_from_file(&parse_session, &path); + let mut parser = new_parser_from_file(&parse_session, &path); let _ = parser.parse_crate_mod(); } diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs index 932a173bc67..99da605f357 100644 --- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs +++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs @@ -21,6 +21,7 @@ extern crate rustc_data_structures; extern crate syntax; +extern crate syntax_expand; use rustc_data_structures::thin_vec::ThinVec; use syntax::ast::*; @@ -28,14 +29,15 @@ use syntax::sess::ParseSess; use syntax::source_map::{Spanned, DUMMY_SP, FileName}; use syntax::source_map::FilePathMapping; use syntax::mut_visit::{self, MutVisitor, visit_clobber}; -use syntax::parse; +use syntax::parse::new_parser_from_source_str; use syntax::print::pprust; use syntax::ptr::P; +use syntax_expand::config::process_configure_mod; fn parse_expr(ps: &ParseSess, src: &str) -> Option> { let src_as_string = src.to_string(); - let mut p = parse::new_parser_from_source_str( + let mut p = new_parser_from_source_str( ps, FileName::Custom(src_as_string.clone()), src_as_string, @@ -202,7 +204,7 @@ fn main() { } fn run() { - let ps = ParseSess::new(FilePathMapping::empty()); + let ps = ParseSess::new(FilePathMapping::empty(), process_configure_mod); iter_exprs(2, &mut |mut e| { // If the pretty printer is correct, then `parse(print(e))` should be identical to `e`, -- cgit 1.4.1-3-g733a5