diff options
Diffstat (limited to 'src/libsyntax')
| -rw-r--r-- | src/libsyntax/ast.rs | 15 | ||||
| -rw-r--r-- | src/libsyntax/ast/tests.rs | 8 | ||||
| -rw-r--r-- | src/libsyntax/lib.rs | 8 | ||||
| -rw-r--r-- | src/libsyntax/mut_visit.rs | 77 | ||||
| -rw-r--r-- | src/libsyntax/mut_visit/tests.rs | 71 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/comments.rs | 54 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/comments/tests.rs | 47 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/mod.rs | 262 | ||||
| -rw-r--r-- | src/libsyntax/parse/lexer/tests.rs | 255 | ||||
| -rw-r--r-- | src/libsyntax/parse/mod.rs | 296 | ||||
| -rw-r--r-- | src/libsyntax/parse/tests.rs | 339 | ||||
| -rw-r--r-- | src/libsyntax/print/pprust.rs | 60 | ||||
| -rw-r--r-- | src/libsyntax/print/pprust/tests.rs | 53 | ||||
| -rw-r--r-- | src/libsyntax/source_map.rs | 223 | ||||
| -rw-r--r-- | src/libsyntax/source_map/tests.rs | 213 | ||||
| -rw-r--r-- | src/libsyntax/tests.rs (renamed from src/libsyntax/test_snippet.rs) | 98 | ||||
| -rw-r--r-- | src/libsyntax/tokenstream.rs | 114 | ||||
| -rw-r--r-- | src/libsyntax/tokenstream/tests.rs | 108 | ||||
| -rw-r--r-- | src/libsyntax/util/lev_distance.rs | 60 | ||||
| -rw-r--r-- | src/libsyntax/util/lev_distance/tests.rs | 58 | ||||
| -rw-r--r-- | src/libsyntax/util/parser_testing.rs | 160 |
21 files changed, 1277 insertions, 1302 deletions
diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index b633705a65f..87113b4b98e 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -27,6 +27,9 @@ use std::fmt; pub use rustc_target::abi::FloatTy; +#[cfg(test)] +mod tests; + #[derive(Clone, RustcEncodable, RustcDecodable, Copy)] pub struct Label { pub ident: Ident, @@ -2432,15 +2435,3 @@ impl ForeignItemKind { } } } - -#[cfg(test)] -mod tests { - use super::*; - - // Are ASTs encodable? - #[test] - fn check_asts_encodable() { - fn assert_encodable<T: rustc_serialize::Encodable>() {} - assert_encodable::<Crate>(); - } -} diff --git a/src/libsyntax/ast/tests.rs b/src/libsyntax/ast/tests.rs new file mode 100644 index 00000000000..7558e9cc3a3 --- /dev/null +++ b/src/libsyntax/ast/tests.rs @@ -0,0 +1,8 @@ +use super::*; + +// Are ASTs encodable? +#[test] +fn check_asts_encodable() { + fn assert_encodable<T: rustc_serialize::Encodable>() {} + assert_encodable::<Crate>(); +} diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 1fd20fa0b31..8ac48d8d74a 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -33,6 +33,9 @@ pub use rustc_data_structures::thin_vec::ThinVec; use ast::AttrId; use syntax_pos::edition::Edition; +#[cfg(test)] +mod tests; + const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments"); // A variant of 'try!' that panics on an Err. This is used as a crutch on the @@ -132,8 +135,6 @@ pub mod util { pub mod lev_distance; pub mod node_count; pub mod parser; - #[cfg(test)] - pub mod parser_testing; pub mod map_in_place; } @@ -183,7 +184,4 @@ pub mod ext { pub mod early_buffered_lints; -#[cfg(test)] -mod test_snippet; - __build_diagnostic_array! { libsyntax, DIAGNOSTICS } diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 7b328e817bf..a5085c5f879 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -22,6 +22,9 @@ use rustc_data_structures::sync::Lrc; use std::ops::DerefMut; use std::{panic, process, ptr}; +#[cfg(test)] +mod tests; + pub trait ExpectOne<A: Array> { fn expect_one(self, err: &'static str) -> A::Item; } @@ -1255,77 +1258,3 @@ pub fn noop_visit_vis<T: MutVisitor>(Spanned { node, span }: &mut Visibility, vi } vis.visit_span(span); } - -#[cfg(test)] -mod tests { - use crate::ast::{self, Ident}; - use crate::util::parser_testing::{string_to_crate, matches_codepattern}; - use crate::print::pprust; - use crate::mut_visit; - use crate::with_default_globals; - use super::*; - - // this version doesn't care about getting comments or docstrings in. - fn fake_print_crate(s: &mut pprust::State<'_>, - krate: &ast::Crate) { - s.print_mod(&krate.module, &krate.attrs) - } - - // change every identifier to "zz" - struct ToZzIdentMutVisitor; - - impl MutVisitor for ToZzIdentMutVisitor { - fn visit_ident(&mut self, ident: &mut ast::Ident) { - *ident = Ident::from_str("zz"); - } - fn visit_mac(&mut self, mac: &mut ast::Mac) { - mut_visit::noop_visit_mac(mac, self) - } - } - - // maybe add to expand.rs... - macro_rules! assert_pred { - ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( - { - let pred_val = $pred; - let a_val = $a; - let b_val = $b; - if !(pred_val(&a_val, &b_val)) { - panic!("expected args satisfying {}, got {} and {}", - $predname, a_val, b_val); - } - } - ) - } - - // make sure idents get transformed everywhere - #[test] fn ident_transformation () { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); - }) - } - - // even inside macro defs.... - #[test] fn ident_transformation_in_defs () { - with_default_globals(|| { - let mut zz_visitor = ToZzIdentMutVisitor; - let mut krate = string_to_crate( - "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ - (g $(d $d $e)+))} ".to_string()); - zz_visitor.visit_crate(&mut krate); - assert_pred!( - matches_codepattern, - "matches_codepattern", - pprust::to_string(|s| fake_print_crate(s, &krate)), - "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string()); - }) - } -} diff --git a/src/libsyntax/mut_visit/tests.rs b/src/libsyntax/mut_visit/tests.rs new file mode 100644 index 00000000000..6868736976b --- /dev/null +++ b/src/libsyntax/mut_visit/tests.rs @@ -0,0 +1,71 @@ +use super::*; + +use crate::ast::{self, Ident}; +use crate::tests::{string_to_crate, matches_codepattern}; +use crate::print::pprust; +use crate::mut_visit; +use crate::with_default_globals; + +// this version doesn't care about getting comments or docstrings in. +fn fake_print_crate(s: &mut pprust::State<'_>, + krate: &ast::Crate) { + s.print_mod(&krate.module, &krate.attrs) +} + +// change every identifier to "zz" +struct ToZzIdentMutVisitor; + +impl MutVisitor for ToZzIdentMutVisitor { + fn visit_ident(&mut self, ident: &mut ast::Ident) { + *ident = Ident::from_str("zz"); + } + fn visit_mac(&mut self, mac: &mut ast::Mac) { + mut_visit::noop_visit_mac(mac, self) + } +} + +// maybe add to expand.rs... +macro_rules! assert_pred { + ($pred:expr, $predname:expr, $a:expr , $b:expr) => ( + { + let pred_val = $pred; + let a_val = $a; + let b_val = $b; + if !(pred_val(&a_val, &b_val)) { + panic!("expected args satisfying {}, got {} and {}", + $predname, a_val, b_val); + } + } + ) +} + +// make sure idents get transformed everywhere +#[test] fn ident_transformation () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string()); + }) +} + +// even inside macro defs.... +#[test] fn ident_transformation_in_defs () { + with_default_globals(|| { + let mut zz_visitor = ToZzIdentMutVisitor; + let mut krate = string_to_crate( + "macro_rules! a {(b $c:expr $(d $e:token)f+ => \ + (g $(d $d $e)+))} ".to_string()); + zz_visitor.visit_crate(&mut krate); + assert_pred!( + matches_codepattern, + "matches_codepattern", + pprust::to_string(|s| fake_print_crate(s, &krate)), + "macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string()); + }) +} diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs index d8f22072d7d..5121a9ef7b5 100644 --- a/src/libsyntax/parse/lexer/comments.rs +++ b/src/libsyntax/parse/lexer/comments.rs @@ -9,6 +9,9 @@ use syntax_pos::{BytePos, CharPos, Pos, FileName}; use std::usize; +#[cfg(test)] +mod tests; + #[derive(Clone, Copy, PartialEq, Debug)] pub enum CommentStyle { /// No code on either side of each line of the comment @@ -249,54 +252,3 @@ pub fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec<Com comments } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_block_doc_comment_1() { - let comment = "/**\n * Test \n ** Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test \n* Test\n Test"); - } - - #[test] - fn test_block_doc_comment_2() { - let comment = "/**\n * Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " Test\n Test"); - } - - #[test] - fn test_block_doc_comment_3() { - let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " let a: *i32;\n *a = 5;"); - } - - #[test] - fn test_block_doc_comment_4() { - let comment = "/*******************\n test\n *********************/"; - let stripped = strip_doc_comment_decoration(comment); - assert_eq!(stripped, " test"); - } - - #[test] - fn test_line_doc_comment() { - let stripped = strip_doc_comment_decoration("/// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///! test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("// test"); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration("///test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("///!test"); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration("//test"); - assert_eq!(stripped, "test"); - } -} diff --git a/src/libsyntax/parse/lexer/comments/tests.rs b/src/libsyntax/parse/lexer/comments/tests.rs new file mode 100644 index 00000000000..f9cd69fb50d --- /dev/null +++ b/src/libsyntax/parse/lexer/comments/tests.rs @@ -0,0 +1,47 @@ +use super::*; + +#[test] +fn test_block_doc_comment_1() { + let comment = "/**\n * Test \n ** Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test \n* Test\n Test"); +} + +#[test] +fn test_block_doc_comment_2() { + let comment = "/**\n * Test\n * Test\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " Test\n Test"); +} + +#[test] +fn test_block_doc_comment_3() { + let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " let a: *i32;\n *a = 5;"); +} + +#[test] +fn test_block_doc_comment_4() { + let comment = "/*******************\n test\n *********************/"; + let stripped = strip_doc_comment_decoration(comment); + assert_eq!(stripped, " test"); +} + +#[test] +fn test_line_doc_comment() { + let stripped = strip_doc_comment_decoration("/// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///! test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("// test"); + assert_eq!(stripped, " test"); + let stripped = strip_doc_comment_decoration("///test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("///!test"); + assert_eq!(stripped, "test"); + let stripped = strip_doc_comment_decoration("//test"); + assert_eq!(stripped, "test"); +} diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 263eb1ac7a4..950b1b2ff53 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -15,6 +15,9 @@ use std::convert::TryInto; use rustc_data_structures::sync::Lrc; use log::debug; +#[cfg(test)] +mod tests; + pub mod comments; mod tokentrees; mod unicode_chars; @@ -777,262 +780,3 @@ fn is_block_doc_comment(s: &str) -> bool { debug!("is {:?} a doc comment? {}", s, res); res } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::ast::CrateConfig; - use crate::symbol::Symbol; - use crate::source_map::{SourceMap, FilePathMapping}; - use crate::feature_gate::UnstableFeatures; - use crate::parse::token; - use crate::diagnostics::plugin::ErrorMap; - use crate::with_default_globals; - use std::io; - use std::path::PathBuf; - use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition}; - use rustc_data_structures::fx::{FxHashSet, FxHashMap}; - use rustc_data_structures::sync::{Lock, Once}; - - fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { - let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), - Some(sm.clone()), - false, - false, - false); - ParseSess { - span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)), - unstable_features: UnstableFeatures::from_environment(), - config: CrateConfig::default(), - included_mod_stack: Lock::new(Vec::new()), - source_map: sm, - missing_fragment_specifiers: Lock::new(FxHashSet::default()), - raw_identifier_spans: Lock::new(Vec::new()), - registered_diagnostics: Lock::new(ErrorMap::new()), - buffered_lints: Lock::new(vec![]), - edition: Edition::from_session(), - ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), - param_attr_spans: Lock::new(Vec::new()), - let_chains_spans: Lock::new(Vec::new()), - async_closure_spans: Lock::new(Vec::new()), - injected_crate_name: Once::new(), - } - } - - // open a string reader for the given string - fn setup<'a>(sm: &SourceMap, - sess: &'a ParseSess, - teststr: String) - -> StringReader<'a> { - let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); - StringReader::new(sess, sf, None) - } - - #[test] - fn t1() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut string_reader = setup(&sm, - &sh, - "/* my source file */ fn main() { println!(\"zebra\"); }\n" - .to_string()); - assert_eq!(string_reader.next_token(), token::Comment); - assert_eq!(string_reader.next_token(), token::Whitespace); - let tok1 = string_reader.next_token(); - let tok2 = Token::new( - mk_ident("fn"), - Span::new(BytePos(21), BytePos(23), NO_EXPANSION), - ); - assert_eq!(tok1.kind, tok2.kind); - assert_eq!(tok1.span, tok2.span); - assert_eq!(string_reader.next_token(), token::Whitespace); - // read another token: - let tok3 = string_reader.next_token(); - assert_eq!(string_reader.pos.clone(), BytePos(28)); - let tok4 = Token::new( - mk_ident("main"), - Span::new(BytePos(24), BytePos(28), NO_EXPANSION), - ); - assert_eq!(tok3.kind, tok4.kind); - assert_eq!(tok3.span, tok4.span); - - assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); - assert_eq!(string_reader.pos.clone(), BytePos(29)) - }) - } - - // check that the given reader produces the desired stream - // of tokens (stop checking after exhausting the expected vec) - fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) { - for expected_tok in &expected { - assert_eq!(&string_reader.next_token(), expected_tok); - } - } - - // make the identifier by looking up the string in the interner - fn mk_ident(id: &str) -> TokenKind { - token::Ident(Symbol::intern(id), false) - } - - fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { - TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) - } - - #[test] - fn doublecolonparsing() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a b".to_string()), - vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_2() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a::b".to_string()), - vec![mk_ident("a"), token::ModSep, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_3() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a ::b".to_string()), - vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); - }) - } - - #[test] - fn dcparsing_4() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - check_tokenization(setup(&sm, &sh, "a:: b".to_string()), - vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); - }) - } - - #[test] - fn character_a() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(), - mk_lit(token::Char, "a", None)); - }) - } - - #[test] - fn character_space() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(), - mk_lit(token::Char, " ", None)); - }) - } - - #[test] - fn character_escaped() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token(), - mk_lit(token::Char, "\\n", None)); - }) - } - - #[test] - fn lifetime_name() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token(), - token::Lifetime(Symbol::intern("'abc"))); - }) - } - - #[test] - fn raw_string() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), - mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None)); - }) - } - - #[test] - fn literal_suffixes() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - macro_rules! test { - ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ - assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, Some("suffix"))); - // with a whitespace separator: - assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token(), - mk_lit(token::$tok_type, $tok_contents, None)); - }} - } - - test!("'a'", Char, "a"); - test!("b'a'", Byte, "a"); - test!("\"a\"", Str, "a"); - test!("b\"a\"", ByteStr, "a"); - test!("1234", Integer, "1234"); - test!("0b101", Integer, "0b101"); - test!("0xABC", Integer, "0xABC"); - test!("1.0", Float, "1.0"); - test!("1.0e10", Float, "1.0e10"); - - assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token(), - mk_lit(token::Integer, "2", Some("us"))); - assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::StrRaw(3), "raw", Some("suffix"))); - assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), - mk_lit(token::ByteStrRaw(3), "raw", Some("suffix"))); - }) - } - - #[test] - fn line_doc_comments() { - assert!(is_doc_comment("///")); - assert!(is_doc_comment("/// blah")); - assert!(!is_doc_comment("////")); - } - - #[test] - fn nested_block_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); - assert_eq!(lexer.next_token(), token::Comment); - assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); - }) - } - - #[test] - fn crlf_comments() { - with_default_globals(|| { - let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); - let sh = mk_sess(sm.clone()); - let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); - let comment = lexer.next_token(); - assert_eq!(comment.kind, token::Comment); - assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); - assert_eq!(lexer.next_token(), token::Whitespace); - assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); - }) - } -} diff --git a/src/libsyntax/parse/lexer/tests.rs b/src/libsyntax/parse/lexer/tests.rs new file mode 100644 index 00000000000..fc47e4f0b18 --- /dev/null +++ b/src/libsyntax/parse/lexer/tests.rs @@ -0,0 +1,255 @@ +use super::*; + +use crate::ast::CrateConfig; +use crate::symbol::Symbol; +use crate::source_map::{SourceMap, FilePathMapping}; +use crate::feature_gate::UnstableFeatures; +use crate::parse::token; +use crate::diagnostics::plugin::ErrorMap; +use crate::with_default_globals; +use std::io; +use std::path::PathBuf; +use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition}; +use rustc_data_structures::fx::{FxHashSet, FxHashMap}; +use rustc_data_structures::sync::{Lock, Once}; + +fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess { + let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()), + Some(sm.clone()), + false, + false, + false); + ParseSess { + span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)), + unstable_features: UnstableFeatures::from_environment(), + config: CrateConfig::default(), + included_mod_stack: Lock::new(Vec::new()), + source_map: sm, + missing_fragment_specifiers: Lock::new(FxHashSet::default()), + raw_identifier_spans: Lock::new(Vec::new()), + registered_diagnostics: Lock::new(ErrorMap::new()), + buffered_lints: Lock::new(vec![]), + edition: Edition::from_session(), + ambiguous_block_expr_parse: Lock::new(FxHashMap::default()), + param_attr_spans: Lock::new(Vec::new()), + let_chains_spans: Lock::new(Vec::new()), + async_closure_spans: Lock::new(Vec::new()), + injected_crate_name: Once::new(), + } +} + +// open a string reader for the given string +fn setup<'a>(sm: &SourceMap, + sess: &'a ParseSess, + teststr: String) + -> StringReader<'a> { + let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr); + StringReader::new(sess, sf, None) +} + +#[test] +fn t1() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut string_reader = setup(&sm, + &sh, + "/* my source file */ fn main() { println!(\"zebra\"); }\n" + .to_string()); + assert_eq!(string_reader.next_token(), token::Comment); + assert_eq!(string_reader.next_token(), token::Whitespace); + let tok1 = string_reader.next_token(); + let tok2 = Token::new( + mk_ident("fn"), + Span::new(BytePos(21), BytePos(23), NO_EXPANSION), + ); + assert_eq!(tok1.kind, tok2.kind); + assert_eq!(tok1.span, tok2.span); + assert_eq!(string_reader.next_token(), token::Whitespace); + // read another token: + let tok3 = string_reader.next_token(); + assert_eq!(string_reader.pos.clone(), BytePos(28)); + let tok4 = Token::new( + mk_ident("main"), + Span::new(BytePos(24), BytePos(28), NO_EXPANSION), + ); + assert_eq!(tok3.kind, tok4.kind); + assert_eq!(tok3.span, tok4.span); + + assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren)); + assert_eq!(string_reader.pos.clone(), BytePos(29)) + }) +} + +// check that the given reader produces the desired stream +// of tokens (stop checking after exhausting the expected vec) +fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) { + for expected_tok in &expected { + assert_eq!(&string_reader.next_token(), expected_tok); + } +} + +// make the identifier by looking up the string in the interner +fn mk_ident(id: &str) -> TokenKind { + token::Ident(Symbol::intern(id), false) +} + +fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { + TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern)) +} + +#[test] +fn doublecolonparsing() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a b".to_string()), + vec![mk_ident("a"), token::Whitespace, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_2() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a::b".to_string()), + vec![mk_ident("a"), token::ModSep, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_3() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a ::b".to_string()), + vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]); + }) +} + +#[test] +fn dcparsing_4() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + check_tokenization(setup(&sm, &sh, "a:: b".to_string()), + vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]); + }) +} + +#[test] +fn character_a() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(), + mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn character_space() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(), + mk_lit(token::Char, " ", None)); + }) +} + +#[test] +fn character_escaped() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token(), + mk_lit(token::Char, "\\n", None)); + }) +} + +#[test] +fn lifetime_name() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token(), + token::Lifetime(Symbol::intern("'abc"))); + }) +} + +#[test] +fn raw_string() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(), + mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None)); + }) +} + +#[test] +fn literal_suffixes() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + macro_rules! test { + ($input: expr, $tok_type: ident, $tok_contents: expr) => {{ + assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, Some("suffix"))); + // with a whitespace separator: + assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token(), + mk_lit(token::$tok_type, $tok_contents, None)); + }} + } + + test!("'a'", Char, "a"); + test!("b'a'", Byte, "a"); + test!("\"a\"", Str, "a"); + test!("b\"a\"", ByteStr, "a"); + test!("1234", Integer, "1234"); + test!("0b101", Integer, "0b101"); + test!("0xABC", Integer, "0xABC"); + test!("1.0", Float, "1.0"); + test!("1.0e10", Float, "1.0e10"); + + assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token(), + mk_lit(token::Integer, "2", Some("us"))); + assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::StrRaw(3), "raw", Some("suffix"))); + assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(), + mk_lit(token::ByteStrRaw(3), "raw", Some("suffix"))); + }) +} + +#[test] +fn line_doc_comments() { + assert!(is_doc_comment("///")); + assert!(is_doc_comment("/// blah")); + assert!(!is_doc_comment("////")); +} + +#[test] +fn nested_block_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string()); + assert_eq!(lexer.next_token(), token::Comment); + assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None)); + }) +} + +#[test] +fn crlf_comments() { + with_default_globals(|| { + let sm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let sh = mk_sess(sm.clone()); + let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string()); + let comment = lexer.next_token(); + assert_eq!(comment.kind, token::Comment); + assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); + assert_eq!(lexer.next_token(), token::Whitespace); + assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); + }) +} diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 1aac8bbb7aa..b7deee688ca 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -22,7 +22,8 @@ use std::borrow::Cow; use std::path::{Path, PathBuf}; use std::str; -pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; +#[cfg(test)] +mod tests; #[macro_use] pub mod parser; @@ -35,6 +36,8 @@ crate mod diagnostics; crate mod literal; crate mod unescape_error_reporting; +pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>; + /// Info about a parsing session. pub struct ParseSess { pub span_diagnostic: Handler, @@ -389,294 +392,3 @@ impl SeqSep { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::ast::{self, Name, PatKind}; - use crate::attr::first_attr_value_str_by_name; - use crate::ptr::P; - use crate::parse::token::Token; - use crate::print::pprust::item_to_string; - use crate::symbol::{kw, sym}; - use crate::tokenstream::{DelimSpan, TokenTree}; - use crate::util::parser_testing::string_to_stream; - use crate::util::parser_testing::{string_to_expr, string_to_item}; - use crate::with_default_globals; - use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; - - /// Parses an item. - /// - /// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` - /// when a syntax error occurred. - fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) - -> PResult<'_, Option<P<ast::Item>>> { - new_parser_from_source_str(sess, name, source).parse_item() - } - - // produce a syntax_pos::span - fn sp(a: u32, b: u32) -> Span { - Span::new(BytePos(a), BytePos(b), NO_EXPANSION) - } - - #[should_panic] - #[test] fn bad_path_expr_1() { - with_default_globals(|| { - string_to_expr("::abc::def::return".to_string()); - }) - } - - // check the token-tree-ization of macros - #[test] - fn string_to_tts_macro () { - with_default_globals(|| { - let tts: Vec<_> = - string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); - let tts: &[TokenTree] = &tts[..]; - - match tts { - [ - TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), - TokenTree::Token(Token { kind: token::Not, .. }), - TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), - TokenTree::Delimited(_, macro_delim, macro_tts) - ] - if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => { - let tts = ¯o_tts.trees().collect::<Vec<_>>(); - match &tts[..] { - [ - TokenTree::Delimited(_, first_delim, first_tts), - TokenTree::Token(Token { kind: token::FatArrow, .. }), - TokenTree::Delimited(_, second_delim, second_tts), - ] - if macro_delim == &token::Paren => { - let tts = &first_tts.trees().collect::<Vec<_>>(); - match &tts[..] { - [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), - ] - if first_delim == &token::Paren && name.as_str() == "a" => {}, - _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), - } - let tts = &second_tts.trees().collect::<Vec<_>>(); - match &tts[..] { - [ - TokenTree::Token(Token { kind: token::Dollar, .. }), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }), - ] - if second_delim == &token::Paren && name.as_str() == "a" => {}, - _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), - } - }, - _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), - } - }, - _ => panic!("value: {:?}",tts), - } - }) - } - - #[test] - fn string_to_tts_1() { - with_default_globals(|| { - let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - - let expected = TokenStream::new(vec![ - TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), - TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(5, 6), sp(13, 14)), - token::DelimToken::Paren, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), - TokenTree::token(token::Colon, sp(8, 9)).into(), - TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), - ]).into(), - ).into(), - TokenTree::Delimited( - DelimSpan::from_pair(sp(15, 16), sp(20, 21)), - token::DelimToken::Brace, - TokenStream::new(vec![ - TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), - TokenTree::token(token::Semi, sp(18, 19)).into(), - ]).into(), - ).into() - ]); - - assert_eq!(tts, expected); - }) - } - - #[test] fn parse_use() { - with_default_globals(|| { - let use_s = "use foo::bar::baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - - let use_s = "use foo::bar as baz;"; - let vitem = string_to_item(use_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], use_s); - }) - } - - #[test] fn parse_extern_crate() { - with_default_globals(|| { - let ex_s = "extern crate foo;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - - let ex_s = "extern crate foo as bar;"; - let vitem = string_to_item(ex_s.to_string()).unwrap(); - let vitem_s = item_to_string(&vitem); - assert_eq!(&vitem_s[..], ex_s); - }) - } - - fn get_spans_of_pat_idents(src: &str) -> Vec<Span> { - let item = string_to_item(src.to_string()).unwrap(); - - struct PatIdentVisitor { - spans: Vec<Span> - } - impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { - fn visit_pat(&mut self, p: &'a ast::Pat) { - match p.node { - PatKind::Ident(_ , ref spannedident, _) => { - self.spans.push(spannedident.span.clone()); - } - _ => { - crate::visit::walk_pat(self, p); - } - } - } - } - let mut v = PatIdentVisitor { spans: Vec::new() }; - crate::visit::walk_item(&mut v, &item); - return v.spans; - } - - #[test] fn span_of_self_arg_pat_idents_are_correct() { - with_default_globals(|| { - - let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", - "impl z { fn a (&mut self, &myarg: i32) {} }", - "impl z { fn a (&'a self, &myarg: i32) {} }", - "impl z { fn a (self, &myarg: i32) {} }", - "impl z { fn a (self: Foo, &myarg: i32) {} }", - ]; - - for &src in &srcs { - let spans = get_spans_of_pat_idents(src); - let (lo, hi) = (spans[0].lo(), spans[0].hi()); - assert!("self" == &src[lo.to_usize()..hi.to_usize()], - "\"{}\" != \"self\". src=\"{}\"", - &src[lo.to_usize()..hi.to_usize()], src) - } - }) - } - - #[test] fn parse_exprs () { - with_default_globals(|| { - // just make sure that they parse.... - string_to_expr("3 + 4".to_string()); - string_to_expr("a::z.froob(b,&(987+3))".to_string()); - }) - } - - #[test] fn attrs_fix_bug () { - with_default_globals(|| { - string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) - -> Result<Box<Writer>, String> { - #[cfg(windows)] - fn wb() -> c_int { - (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int - } - - #[cfg(unix)] - fn wb() -> c_int { O_WRONLY as c_int } - - let mut fflags: c_int = wb(); -}".to_string()); - }) - } - - #[test] fn crlf_doc_comments() { - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - - let name_1 = FileName::Custom("crlf_source_1".to_string()); - let source = "/// doc comment\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_1, source, &sess) - .unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); - assert_eq!(doc.as_str(), "/// doc comment"); - - let name_2 = FileName::Custom("crlf_source_2".to_string()); - let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_2, source, &sess) - .unwrap().unwrap(); - let docs = item.attrs.iter().filter(|a| a.path == sym::doc) - .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>(); - let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; - assert_eq!(&docs[..], b); - - let name_3 = FileName::Custom("clrf_source_3".to_string()); - let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); - let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); - let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); - assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); - }); - } - - #[test] - fn ttdelim_span() { - fn parse_expr_from_source_str( - name: FileName, source: String, sess: &ParseSess - ) -> PResult<'_, P<ast::Expr>> { - new_parser_from_source_str(sess, name, source).parse_expr() - } - - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), - "foo!( fn main() { body } )".to_string(), &sess).unwrap(); - - let tts: Vec<_> = match expr.node { - ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(), - _ => panic!("not a macro"), - }; - - let span = tts.iter().rev().next().unwrap().span(); - - match sess.source_map().span_to_snippet(span) { - Ok(s) => assert_eq!(&s[..], "{ body }"), - Err(_) => panic!("could not get snippet"), - } - }); - } - - // This tests that when parsing a string (rather than a file) we don't try - // and read in a file for a module declaration and just parse a stub. - // See `recurse_into_file_modules` in the parser. - #[test] - fn out_of_line_mod() { - with_default_globals(|| { - let sess = ParseSess::new(FilePathMapping::empty()); - let item = parse_item_from_source_str( - PathBuf::from("foo").into(), - "mod foo { struct S; mod this_does_not_exist; }".to_owned(), - &sess, - ).unwrap().unwrap(); - - if let ast::ItemKind::Mod(ref m) = item.node { - assert!(m.items.len() == 2); - } else { - panic!(); - } - }); - } -} diff --git a/src/libsyntax/parse/tests.rs b/src/libsyntax/parse/tests.rs new file mode 100644 index 00000000000..e619fd17fb5 --- /dev/null +++ b/src/libsyntax/parse/tests.rs @@ -0,0 +1,339 @@ +use super::*; + +use crate::ast::{self, Name, PatKind}; +use crate::attr::first_attr_value_str_by_name; +use crate::parse::{ParseSess, PResult}; +use crate::parse::new_parser_from_source_str; +use crate::parse::token::Token; +use crate::print::pprust::item_to_string; +use crate::ptr::P; +use crate::source_map::FilePathMapping; +use crate::symbol::{kw, sym}; +use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse}; +use crate::tokenstream::{DelimSpan, TokenTree, TokenStream}; +use crate::with_default_globals; +use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION}; + +use std::path::PathBuf; + +/// Parses an item. +/// +/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err` +/// when a syntax error occurred. +fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess) + -> PResult<'_, Option<P<ast::Item>>> { + new_parser_from_source_str(sess, name, source).parse_item() +} + +// produce a syntax_pos::span +fn sp(a: u32, b: u32) -> Span { + Span::new(BytePos(a), BytePos(b), NO_EXPANSION) +} + +/// Parse a string, return an expr +fn string_to_expr(source_str : String) -> P<ast::Expr> { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_expr() + }) +} + +/// Parse a string, return an item +fn string_to_item(source_str : String) -> Option<P<ast::Item>> { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_item() + }) +} + +#[should_panic] +#[test] fn bad_path_expr_1() { + with_default_globals(|| { + string_to_expr("::abc::def::return".to_string()); + }) +} + +// check the token-tree-ization of macros +#[test] +fn string_to_tts_macro () { + with_default_globals(|| { + let tts: Vec<_> = + string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect(); + let tts: &[TokenTree] = &tts[..]; + + match tts { + [ + TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), + TokenTree::Token(Token { kind: token::Not, .. }), + TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), + TokenTree::Delimited(_, macro_delim, macro_tts) + ] + if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => { + let tts = ¯o_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Delimited(_, first_delim, first_tts), + TokenTree::Token(Token { kind: token::FatArrow, .. }), + TokenTree::Delimited(_, second_delim, second_tts), + ] + if macro_delim == &token::Paren => { + let tts = &first_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if first_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), + } + let tts = &second_tts.trees().collect::<Vec<_>>(); + match &tts[..] { + [ + TokenTree::Token(Token { kind: token::Dollar, .. }), + TokenTree::Token(Token { kind: token::Ident(name, false), .. }), + ] + if second_delim == &token::Paren && name.as_str() == "a" => {}, + _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), + } + }, + _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), + } + }, + _ => panic!("value: {:?}",tts), + } + }) +} + +#[test] +fn string_to_tts_1() { + with_default_globals(|| { + let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); + + let expected = TokenStream::new(vec![ + TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), + TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(5, 6), sp(13, 14)), + token::DelimToken::Paren, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(), + TokenTree::token(token::Colon, sp(8, 9)).into(), + TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), + ]).into(), + ).into(), + TokenTree::Delimited( + DelimSpan::from_pair(sp(15, 16), sp(20, 21)), + token::DelimToken::Brace, + TokenStream::new(vec![ + TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(), + TokenTree::token(token::Semi, sp(18, 19)).into(), + ]).into(), + ).into() + ]); + + assert_eq!(tts, expected); + }) +} + +#[test] fn parse_use() { + with_default_globals(|| { + let use_s = "use foo::bar::baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + + let use_s = "use foo::bar as baz;"; + let vitem = string_to_item(use_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], use_s); + }) +} + +#[test] fn parse_extern_crate() { + with_default_globals(|| { + let ex_s = "extern crate foo;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + + let ex_s = "extern crate foo as bar;"; + let vitem = string_to_item(ex_s.to_string()).unwrap(); + let vitem_s = item_to_string(&vitem); + assert_eq!(&vitem_s[..], ex_s); + }) +} + +fn get_spans_of_pat_idents(src: &str) -> Vec<Span> { + let item = string_to_item(src.to_string()).unwrap(); + + struct PatIdentVisitor { + spans: Vec<Span> + } + impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor { + fn visit_pat(&mut self, p: &'a ast::Pat) { + match p.node { + PatKind::Ident(_ , ref spannedident, _) => { + self.spans.push(spannedident.span.clone()); + } + _ => { + crate::visit::walk_pat(self, p); + } + } + } + } + let mut v = PatIdentVisitor { spans: Vec::new() }; + crate::visit::walk_item(&mut v, &item); + return v.spans; +} + +#[test] fn span_of_self_arg_pat_idents_are_correct() { + with_default_globals(|| { + + let srcs = ["impl z { fn a (&self, &myarg: i32) {} }", + "impl z { fn a (&mut self, &myarg: i32) {} }", + "impl z { fn a (&'a self, &myarg: i32) {} }", + "impl z { fn a (self, &myarg: i32) {} }", + "impl z { fn a (self: Foo, &myarg: i32) {} }", + ]; + + for &src in &srcs { + let spans = get_spans_of_pat_idents(src); + let (lo, hi) = (spans[0].lo(), spans[0].hi()); + assert!("self" == &src[lo.to_usize()..hi.to_usize()], + "\"{}\" != \"self\". src=\"{}\"", + &src[lo.to_usize()..hi.to_usize()], src) + } + }) +} + +#[test] fn parse_exprs () { + with_default_globals(|| { + // just make sure that they parse.... + string_to_expr("3 + 4".to_string()); + string_to_expr("a::z.froob(b,&(987+3))".to_string()); + }) +} + +#[test] fn attrs_fix_bug () { + with_default_globals(|| { + string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag]) + -> Result<Box<Writer>, String> { +#[cfg(windows)] +fn wb() -> c_int { + (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int +} + +#[cfg(unix)] +fn wb() -> c_int { O_WRONLY as c_int } + +let mut fflags: c_int = wb(); +}".to_string()); + }) +} + +#[test] fn crlf_doc_comments() { + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + + let name_1 = FileName::Custom("crlf_source_1".to_string()); + let source = "/// doc comment\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_1, source, &sess) + .unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/// doc comment"); + + let name_2 = FileName::Custom("crlf_source_2".to_string()); + let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_2, source, &sess) + .unwrap().unwrap(); + let docs = item.attrs.iter().filter(|a| a.path == sym::doc) + .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>(); + let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()]; + assert_eq!(&docs[..], b); + + let name_3 = FileName::Custom("clrf_source_3".to_string()); + let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); + let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); + let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap(); + assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); + }); +} + +#[test] +fn ttdelim_span() { + fn parse_expr_from_source_str( + name: FileName, source: String, sess: &ParseSess + ) -> PResult<'_, P<ast::Expr>> { + new_parser_from_source_str(sess, name, source).parse_expr() + } + + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + let expr = parse_expr_from_source_str(PathBuf::from("foo").into(), + "foo!( fn main() { body } )".to_string(), &sess).unwrap(); + + let tts: Vec<_> = match expr.node { + ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(), + _ => panic!("not a macro"), + }; + + let span = tts.iter().rev().next().unwrap().span(); + + match sess.source_map().span_to_snippet(span) { + Ok(s) => assert_eq!(&s[..], "{ body }"), + Err(_) => panic!("could not get snippet"), + } + }); +} + +// This tests that when parsing a string (rather than a file) we don't try +// and read in a file for a module declaration and just parse a stub. +// See `recurse_into_file_modules` in the parser. +#[test] +fn out_of_line_mod() { + with_default_globals(|| { + let sess = ParseSess::new(FilePathMapping::empty()); + let item = parse_item_from_source_str( + PathBuf::from("foo").into(), + "mod foo { struct S; mod this_does_not_exist; }".to_owned(), + &sess, + ).unwrap().unwrap(); + + if let ast::ItemKind::Mod(ref m) = item.node { + assert!(m.items.len() == 2); + } else { + panic!(); + } + }); +} + +#[test] +fn eqmodws() { + assert_eq!(matches_codepattern("",""),true); + assert_eq!(matches_codepattern("","a"),false); + assert_eq!(matches_codepattern("a",""),false); + assert_eq!(matches_codepattern("a","a"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); + assert_eq!(matches_codepattern("a b","a b"),true); + assert_eq!(matches_codepattern("ab","a b"),false); + assert_eq!(matches_codepattern("a b","ab"),true); + assert_eq!(matches_codepattern(" a b","ab"),true); +} + +#[test] +fn pattern_whitespace() { + assert_eq!(matches_codepattern("","\x0C"), false); + assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); + assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); +} + +#[test] +fn non_pattern_whitespace() { + // These have the property 'White_Space' but not 'Pattern_White_Space' + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); + assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); + assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); +} diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 88ff6ee9071..f6dd95a7f4f 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -19,6 +19,9 @@ use syntax_pos::{DUMMY_SP, FileName, Span}; use std::borrow::Cow; +#[cfg(test)] +mod tests; + pub enum MacHeader<'a> { Path(&'a ast::Path), Keyword(&'static str), @@ -2888,60 +2891,3 @@ impl<'a> State<'a> { } } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate::ast; - use crate::source_map; - use crate::with_default_globals; - use syntax_pos; - - #[test] - fn test_fun_to_string() { - with_default_globals(|| { - let abba_ident = ast::Ident::from_str("abba"); - - let decl = ast::FnDecl { - inputs: Vec::new(), - output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP), - c_variadic: false - }; - let generics = ast::Generics::default(); - assert_eq!( - fun_to_string( - &decl, - ast::FnHeader { - unsafety: ast::Unsafety::Normal, - constness: source_map::dummy_spanned(ast::Constness::NotConst), - asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync), - abi: Abi::Rust, - }, - abba_ident, - &generics - ), - "fn abba()" - ); - }) - } - - #[test] - fn test_variant_to_string() { - with_default_globals(|| { - let ident = ast::Ident::from_str("principal_skinner"); - - let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ { - ident, - attrs: Vec::new(), - id: ast::DUMMY_NODE_ID, - // making this up as I go.... ? - data: ast::VariantData::Unit(ast::DUMMY_NODE_ID), - disr_expr: None, - }); - - let varstr = variant_to_string(&var); - assert_eq!(varstr, "principal_skinner"); - }) - } -} diff --git a/src/libsyntax/print/pprust/tests.rs b/src/libsyntax/print/pprust/tests.rs new file mode 100644 index 00000000000..97df7e6dcbd --- /dev/null +++ b/src/libsyntax/print/pprust/tests.rs @@ -0,0 +1,53 @@ +use super::*; + +use crate::ast; +use crate::source_map; +use crate::with_default_globals; +use syntax_pos; + +#[test] +fn test_fun_to_string() { + with_default_globals(|| { + let abba_ident = ast::Ident::from_str("abba"); + + let decl = ast::FnDecl { + inputs: Vec::new(), + output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP), + c_variadic: false + }; + let generics = ast::Generics::default(); + assert_eq!( + fun_to_string( + &decl, + ast::FnHeader { + unsafety: ast::Unsafety::Normal, + constness: source_map::dummy_spanned(ast::Constness::NotConst), + asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync), + abi: Abi::Rust, + }, + abba_ident, + &generics + ), + "fn abba()" + ); + }) +} + +#[test] +fn test_variant_to_string() { + with_default_globals(|| { + let ident = ast::Ident::from_str("principal_skinner"); + + let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ { + ident, + attrs: Vec::new(), + id: ast::DUMMY_NODE_ID, + // making this up as I go.... ? + data: ast::VariantData::Unit(ast::DUMMY_NODE_ID), + disr_expr: None, + }); + + let varstr = variant_to_string(&var); + assert_eq!(varstr, "principal_skinner"); + }) +} diff --git a/src/libsyntax/source_map.rs b/src/libsyntax/source_map.rs index bbf62ef1e23..f83c1dbf7ee 100644 --- a/src/libsyntax/source_map.rs +++ b/src/libsyntax/source_map.rs @@ -24,6 +24,9 @@ use log::debug; use errors::SourceMapper; +#[cfg(test)] +mod tests; + /// Returns the span itself if it doesn't come from a macro expansion, /// otherwise return the call site span up to the `enclosing_sp` by /// following the `expn_info` chain. @@ -1020,223 +1023,3 @@ impl FilePathMapping { (path, false) } } - -// _____________________________________________________________________________ -// Tests -// - -#[cfg(test)] -mod tests { - use super::*; - use rustc_data_structures::sync::Lrc; - - fn init_source_map() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - sm.new_source_file(PathBuf::from("blork.rs").into(), - "first line.\nsecond line".to_string()); - sm.new_source_file(PathBuf::from("empty.rs").into(), - String::new()); - sm.new_source_file(PathBuf::from("blork2.rs").into(), - "first line.\nsecond line".to_string()); - sm - } - - #[test] - fn t3() { - // Test lookup_byte_offset - let sm = init_source_map(); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(23)); - - let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); - assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); - assert_eq!(srcfbp1.pos, BytePos(0)); - - let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); - assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); - assert_eq!(srcfbp2.pos, BytePos(0)); - } - - #[test] - fn t4() { - // Test bytepos_to_file_charpos - let sm = init_source_map(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); - assert_eq!(cp1, CharPos(22)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); - assert_eq!(cp2, CharPos(0)); - } - - #[test] - fn t5() { - // Test zero-length source_files. - let sm = init_source_map(); - - let loc1 = sm.lookup_char_pos(BytePos(22)); - assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(loc1.line, 2); - assert_eq!(loc1.col, CharPos(10)); - - let loc2 = sm.lookup_char_pos(BytePos(25)); - assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); - assert_eq!(loc2.line, 1); - assert_eq!(loc2.col, CharPos(0)); - } - - fn init_source_map_mbc() -> SourceMap { - let sm = SourceMap::new(FilePathMapping::empty()); - // € is a three byte utf8 char. - sm.new_source_file(PathBuf::from("blork.rs").into(), - "fir€st €€€€ line.\nsecond line".to_string()); - sm.new_source_file(PathBuf::from("blork2.rs").into(), - "first line€€.\n€ second line".to_string()); - sm - } - - #[test] - fn t6() { - // Test bytepos_to_file_charpos in the presence of multi-byte chars - let sm = init_source_map_mbc(); - - let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); - assert_eq!(cp1, CharPos(3)); - - let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); - assert_eq!(cp2, CharPos(4)); - - let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); - assert_eq!(cp3, CharPos(12)); - - let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); - assert_eq!(cp4, CharPos(15)); - } - - #[test] - fn t7() { - // Test span_to_lines for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let file_lines = sm.span_to_lines(span).unwrap(); - - assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); - assert_eq!(file_lines.lines.len(), 1); - assert_eq!(file_lines.lines[0].line_index, 1); - } - - /// Given a string like " ~~~~~~~~~~~~ ", produces a span - /// converting that range. The idea is that the string has the same - /// length as the input, and we uncover the byte positions. Note - /// that this can span lines and so on. - fn span_from_selection(input: &str, selection: &str) -> Span { - assert_eq!(input.len(), selection.len()); - let left_index = selection.find('~').unwrap() as u32; - let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index); - Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION) - } - - /// Tests span_to_snippet and span_to_lines for a span converting 3 - /// lines in the middle of a file. - #[test] - fn span_to_snippet_and_lines_spanning_multiple_lines() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; - let selection = " \n ~~\n~~~\n~~~~~ \n \n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); - let span = span_from_selection(inputtext, selection); - - // check that we are extracting the text we thought we were extracting - assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); - - // check that span_to_lines gives us the complete result with the lines/cols we expected - let lines = sm.span_to_lines(span).unwrap(); - let expected = vec![ - LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, - LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, - LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) } - ]; - assert_eq!(lines.lines, expected); - } - - #[test] - fn t8() { - // Test span_to_snippet for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let snippet = sm.span_to_snippet(span); - - assert_eq!(snippet, Ok("second line".to_string())); - } - - #[test] - fn t9() { - // Test span_to_str for a span ending at the end of source_file - let sm = init_source_map(); - let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); - let sstr = sm.span_to_string(span); - - assert_eq!(sstr, "blork.rs:2:1: 2:12"); - } - - /// Tests failing to merge two spans on different lines - #[test] - fn span_merging_fail() { - let sm = SourceMap::new(FilePathMapping::empty()); - let inputtext = "bbbb BB\ncc CCC\n"; - let selection1 = " ~~\n \n"; - let selection2 = " \n ~~~\n"; - sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); - let span1 = span_from_selection(inputtext, selection1); - let span2 = span_from_selection(inputtext, selection2); - - assert!(sm.merge_spans(span1, span2).is_none()); - } - - /// Returns the span corresponding to the `n`th occurrence of - /// `substring` in `source_text`. - trait SourceMapExtension { - fn span_substr(&self, - file: &Lrc<SourceFile>, - source_text: &str, - substring: &str, - n: usize) - -> Span; - } - - impl SourceMapExtension for SourceMap { - fn span_substr(&self, - file: &Lrc<SourceFile>, - source_text: &str, - substring: &str, - n: usize) - -> Span - { - println!("span_substr(file={:?}/{:?}, substring={:?}, n={})", - file.name, file.start_pos, substring, n); - let mut i = 0; - let mut hi = 0; - loop { - let offset = source_text[hi..].find(substring).unwrap_or_else(|| { - panic!("source_text `{}` does not have {} occurrences of `{}`, only {}", - source_text, n, substring, i); - }); - let lo = hi + offset; - hi = lo + substring.len(); - if i == n { - let span = Span::new( - BytePos(lo as u32 + file.start_pos.0), - BytePos(hi as u32 + file.start_pos.0), - NO_EXPANSION, - ); - assert_eq!(&self.span_to_snippet(span).unwrap()[..], - substring); - return span; - } - i += 1; - } - } - } -} diff --git a/src/libsyntax/source_map/tests.rs b/src/libsyntax/source_map/tests.rs new file mode 100644 index 00000000000..427e86b56e1 --- /dev/null +++ b/src/libsyntax/source_map/tests.rs @@ -0,0 +1,213 @@ +use super::*; + +use rustc_data_structures::sync::Lrc; + +fn init_source_map() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + sm.new_source_file(PathBuf::from("blork.rs").into(), + "first line.\nsecond line".to_string()); + sm.new_source_file(PathBuf::from("empty.rs").into(), + String::new()); + sm.new_source_file(PathBuf::from("blork2.rs").into(), + "first line.\nsecond line".to_string()); + sm +} + +#[test] +fn t3() { + // Test lookup_byte_offset + let sm = init_source_map(); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(23)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(23)); + + let srcfbp1 = sm.lookup_byte_offset(BytePos(24)); + assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into()); + assert_eq!(srcfbp1.pos, BytePos(0)); + + let srcfbp2 = sm.lookup_byte_offset(BytePos(25)); + assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into()); + assert_eq!(srcfbp2.pos, BytePos(0)); +} + +#[test] +fn t4() { + // Test bytepos_to_file_charpos + let sm = init_source_map(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(22)); + assert_eq!(cp1, CharPos(22)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(25)); + assert_eq!(cp2, CharPos(0)); +} + +#[test] +fn t5() { + // Test zero-length source_files. + let sm = init_source_map(); + + let loc1 = sm.lookup_char_pos(BytePos(22)); + assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(loc1.line, 2); + assert_eq!(loc1.col, CharPos(10)); + + let loc2 = sm.lookup_char_pos(BytePos(25)); + assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into()); + assert_eq!(loc2.line, 1); + assert_eq!(loc2.col, CharPos(0)); +} + +fn init_source_map_mbc() -> SourceMap { + let sm = SourceMap::new(FilePathMapping::empty()); + // € is a three byte utf8 char. + sm.new_source_file(PathBuf::from("blork.rs").into(), + "fir€st €€€€ line.\nsecond line".to_string()); + sm.new_source_file(PathBuf::from("blork2.rs").into(), + "first line€€.\n€ second line".to_string()); + sm +} + +#[test] +fn t6() { + // Test bytepos_to_file_charpos in the presence of multi-byte chars + let sm = init_source_map_mbc(); + + let cp1 = sm.bytepos_to_file_charpos(BytePos(3)); + assert_eq!(cp1, CharPos(3)); + + let cp2 = sm.bytepos_to_file_charpos(BytePos(6)); + assert_eq!(cp2, CharPos(4)); + + let cp3 = sm.bytepos_to_file_charpos(BytePos(56)); + assert_eq!(cp3, CharPos(12)); + + let cp4 = sm.bytepos_to_file_charpos(BytePos(61)); + assert_eq!(cp4, CharPos(15)); +} + +#[test] +fn t7() { + // Test span_to_lines for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let file_lines = sm.span_to_lines(span).unwrap(); + + assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into()); + assert_eq!(file_lines.lines.len(), 1); + assert_eq!(file_lines.lines[0].line_index, 1); +} + +/// Given a string like " ~~~~~~~~~~~~ ", produces a span +/// converting that range. The idea is that the string has the same +/// length as the input, and we uncover the byte positions. Note +/// that this can span lines and so on. +fn span_from_selection(input: &str, selection: &str) -> Span { + assert_eq!(input.len(), selection.len()); + let left_index = selection.find('~').unwrap() as u32; + let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index); + Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION) +} + +/// Tests span_to_snippet and span_to_lines for a span converting 3 +/// lines in the middle of a file. +#[test] +fn span_to_snippet_and_lines_spanning_multiple_lines() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n"; + let selection = " \n ~~\n~~~\n~~~~~ \n \n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string()); + let span = span_from_selection(inputtext, selection); + + // check that we are extracting the text we thought we were extracting + assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD"); + + // check that span_to_lines gives us the complete result with the lines/cols we expected + let lines = sm.span_to_lines(span).unwrap(); + let expected = vec![ + LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) }, + LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) }, + LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) } + ]; + assert_eq!(lines.lines, expected); +} + +#[test] +fn t8() { + // Test span_to_snippet for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let snippet = sm.span_to_snippet(span); + + assert_eq!(snippet, Ok("second line".to_string())); +} + +#[test] +fn t9() { + // Test span_to_str for a span ending at the end of source_file + let sm = init_source_map(); + let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION); + let sstr = sm.span_to_string(span); + + assert_eq!(sstr, "blork.rs:2:1: 2:12"); +} + +/// Tests failing to merge two spans on different lines +#[test] +fn span_merging_fail() { + let sm = SourceMap::new(FilePathMapping::empty()); + let inputtext = "bbbb BB\ncc CCC\n"; + let selection1 = " ~~\n \n"; + let selection2 = " \n ~~~\n"; + sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned()); + let span1 = span_from_selection(inputtext, selection1); + let span2 = span_from_selection(inputtext, selection2); + + assert!(sm.merge_spans(span1, span2).is_none()); +} + +/// Returns the span corresponding to the `n`th occurrence of +/// `substring` in `source_text`. +trait SourceMapExtension { + fn span_substr(&self, + file: &Lrc<SourceFile>, + source_text: &str, + substring: &str, + n: usize) + -> Span; +} + +impl SourceMapExtension for SourceMap { + fn span_substr(&self, + file: &Lrc<SourceFile>, + source_text: &str, + substring: &str, + n: usize) + -> Span + { + println!("span_substr(file={:?}/{:?}, substring={:?}, n={})", + file.name, file.start_pos, substring, n); + let mut i = 0; + let mut hi = 0; + loop { + let offset = source_text[hi..].find(substring).unwrap_or_else(|| { + panic!("source_text `{}` does not have {} occurrences of `{}`, only {}", + source_text, n, substring, i); + }); + let lo = hi + offset; + hi = lo + substring.len(); + if i == n { + let span = Span::new( + BytePos(lo as u32 + file.start_pos.0), + BytePos(hi as u32 + file.start_pos.0), + NO_EXPANSION, + ); + assert_eq!(&self.span_to_snippet(span).unwrap()[..], + substring); + return span; + } + i += 1; + } + } +} diff --git a/src/libsyntax/test_snippet.rs b/src/libsyntax/tests.rs index 107cbe70a23..cff034fdeb1 100644 --- a/src/libsyntax/test_snippet.rs +++ b/src/libsyntax/tests.rs @@ -1,16 +1,106 @@ +use crate::{ast, panictry}; +use crate::parse::{ParseSess, PResult, source_file_to_stream}; +use crate::parse::new_parser_from_source_str; +use crate::parse::parser::Parser; use crate::source_map::{SourceMap, FilePathMapping}; +use crate::tokenstream::TokenStream; use crate::with_default_globals; -use errors::Handler; use errors::emitter::EmitterWriter; +use errors::Handler; +use rustc_data_structures::sync::Lrc; +use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan}; use std::io; use std::io::prelude::*; -use rustc_data_structures::sync::Lrc; +use std::iter::Peekable; +use std::path::{Path, PathBuf}; use std::str; use std::sync::{Arc, Mutex}; -use std::path::Path; -use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan}; + +/// Map string to parser (via tts) +fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { + new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) +} + +crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where + F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, +{ + let mut p = string_to_parser(&ps, s); + let x = panictry!(f(&mut p)); + p.sess.span_diagnostic.abort_if_errors(); + x +} + +/// Map a string to tts, using a made-up filename: +crate fn string_to_stream(source_str: String) -> TokenStream { + let ps = ParseSess::new(FilePathMapping::empty()); + source_file_to_stream( + &ps, + ps.source_map().new_source_file(PathBuf::from("bogofile").into(), + source_str, + ), None).0 +} + +/// Parse a string, return a crate. +crate fn string_to_crate(source_str : String) -> ast::Crate { + let ps = ParseSess::new(FilePathMapping::empty()); + with_error_checking_parse(source_str, &ps, |p| { + p.parse_crate_mod() + }) +} + +/// Does the given string match the pattern? whitespace in the first string +/// may be deleted or replaced with other whitespace to match the pattern. +/// This function is relatively Unicode-ignorant; fortunately, the careful design +/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). +crate fn matches_codepattern(a : &str, b : &str) -> bool { + let mut a_iter = a.chars().peekable(); + let mut b_iter = b.chars().peekable(); + + loop { + let (a, b) = match (a_iter.peek(), b_iter.peek()) { + (None, None) => return true, + (None, _) => return false, + (Some(&a), None) => { + if is_pattern_whitespace(a) { + break // trailing whitespace check is out of loop for borrowck + } else { + return false + } + } + (Some(&a), Some(&b)) => (a, b) + }; + + if is_pattern_whitespace(a) && is_pattern_whitespace(b) { + // skip whitespace for a and b + scan_for_non_ws_or_end(&mut a_iter); + scan_for_non_ws_or_end(&mut b_iter); + } else if is_pattern_whitespace(a) { + // skip whitespace for a + scan_for_non_ws_or_end(&mut a_iter); + } else if a == b { + a_iter.next(); + b_iter.next(); + } else { + return false + } + } + + // check if a has *only* trailing whitespace + a_iter.all(is_pattern_whitespace) +} + +/// Advances the given peekable `Iterator` until it reaches a non-whitespace character +fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) { + while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) { + iter.next(); + } +} + +fn is_pattern_whitespace(c: char) -> bool { + rustc_lexer::character_properties::is_whitespace(c) +} /// Identify a position in the text by the Nth occurrence of a string. struct Position { diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 34e68944926..6ff8898fe21 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -29,6 +29,9 @@ use smallvec::{SmallVec, smallvec}; use std::borrow::Cow; use std::{fmt, iter, mem}; +#[cfg(test)] +mod tests; + /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -552,114 +555,3 @@ impl DelimSpan { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::ast::Name; - use crate::with_default_globals; - use crate::util::parser_testing::string_to_stream; - use syntax_pos::{Span, BytePos, NO_EXPANSION}; - - fn string_to_ts(string: &str) -> TokenStream { - string_to_stream(string.to_owned()) - } - - fn sp(a: u32, b: u32) -> Span { - Span::new(BytePos(a), BytePos(b), NO_EXPANSION) - } - - #[test] - fn test_concat() { - with_default_globals(|| { - let test_res = string_to_ts("foo::bar::baz"); - let test_fst = string_to_ts("foo::bar"); - let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); - assert_eq!(test_res.trees().count(), 5); - assert_eq!(eq_res.trees().count(), 5); - assert_eq!(test_res.eq_unspanned(&eq_res), true); - }) - } - - #[test] - fn test_to_from_bijection() { - with_default_globals(|| { - let test_start = string_to_ts("foo::bar(baz)"); - let test_end = test_start.trees().collect(); - assert_eq!(test_start, test_end) - }) - } - - #[test] - fn test_eq_0() { - with_default_globals(|| { - let test_res = string_to_ts("foo"); - let test_eqs = string_to_ts("foo"); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_eq_1() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("::bar::baz"); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_eq_3() { - with_default_globals(|| { - let test_res = string_to_ts(""); - let test_eqs = string_to_ts(""); - assert_eq!(test_res, test_eqs) - }) - } - - #[test] - fn test_diseq_0() { - with_default_globals(|| { - let test_res = string_to_ts("::bar::baz"); - let test_eqs = string_to_ts("bar::baz"); - assert_eq!(test_res == test_eqs, false) - }) - } - - #[test] - fn test_diseq_1() { - with_default_globals(|| { - let test_res = string_to_ts("(bar,baz)"); - let test_eqs = string_to_ts("bar,baz"); - assert_eq!(test_res == test_eqs, false) - }) - } - - #[test] - fn test_is_empty() { - with_default_globals(|| { - let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); - let test1: TokenStream = - TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); - let test2 = string_to_ts("foo(bar::baz)"); - - assert_eq!(test0.is_empty(), true); - assert_eq!(test1.is_empty(), false); - assert_eq!(test2.is_empty(), false); - }) - } - - #[test] - fn test_dotdotdot() { - with_default_globals(|| { - let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); - builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); - let stream = builder.build(); - assert!(stream.eq_unspanned(&string_to_ts("..."))); - assert_eq!(stream.trees().count(), 1); - }) - } -} diff --git a/src/libsyntax/tokenstream/tests.rs b/src/libsyntax/tokenstream/tests.rs new file mode 100644 index 00000000000..72e22a49876 --- /dev/null +++ b/src/libsyntax/tokenstream/tests.rs @@ -0,0 +1,108 @@ +use super::*; + +use crate::ast::Name; +use crate::with_default_globals; +use crate::tests::string_to_stream; +use syntax_pos::{Span, BytePos, NO_EXPANSION}; + +fn string_to_ts(string: &str) -> TokenStream { + string_to_stream(string.to_owned()) +} + +fn sp(a: u32, b: u32) -> Span { + Span::new(BytePos(a), BytePos(b), NO_EXPANSION) +} + +#[test] +fn test_concat() { + with_default_globals(|| { + let test_res = string_to_ts("foo::bar::baz"); + let test_fst = string_to_ts("foo::bar"); + let test_snd = string_to_ts("::baz"); + let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + assert_eq!(test_res.trees().count(), 5); + assert_eq!(eq_res.trees().count(), 5); + assert_eq!(test_res.eq_unspanned(&eq_res), true); + }) +} + +#[test] +fn test_to_from_bijection() { + with_default_globals(|| { + let test_start = string_to_ts("foo::bar(baz)"); + let test_end = test_start.trees().collect(); + assert_eq!(test_start, test_end) + }) +} + +#[test] +fn test_eq_0() { + with_default_globals(|| { + let test_res = string_to_ts("foo"); + let test_eqs = string_to_ts("foo"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_1() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("::bar::baz"); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_eq_3() { + with_default_globals(|| { + let test_res = string_to_ts(""); + let test_eqs = string_to_ts(""); + assert_eq!(test_res, test_eqs) + }) +} + +#[test] +fn test_diseq_0() { + with_default_globals(|| { + let test_res = string_to_ts("::bar::baz"); + let test_eqs = string_to_ts("bar::baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_diseq_1() { + with_default_globals(|| { + let test_res = string_to_ts("(bar,baz)"); + let test_eqs = string_to_ts("bar,baz"); + assert_eq!(test_res == test_eqs, false) + }) +} + +#[test] +fn test_is_empty() { + with_default_globals(|| { + let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); + let test1: TokenStream = + TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into(); + let test2 = string_to_ts("foo(bar::baz)"); + + assert_eq!(test0.is_empty(), true); + assert_eq!(test1.is_empty(), false); + assert_eq!(test2.is_empty(), false); + }) +} + +#[test] +fn test_dotdotdot() { + with_default_globals(|| { + let mut builder = TokenStreamBuilder::new(); + builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); + builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); + builder.push(TokenTree::token(token::Dot, sp(2, 3))); + let stream = builder.build(); + assert!(stream.eq_unspanned(&string_to_ts("..."))); + assert_eq!(stream.trees().count(), 1); + }) +} diff --git a/src/libsyntax/util/lev_distance.rs b/src/libsyntax/util/lev_distance.rs index 885b5a4f333..4127a8c7fce 100644 --- a/src/libsyntax/util/lev_distance.rs +++ b/src/libsyntax/util/lev_distance.rs @@ -1,6 +1,9 @@ use std::cmp; use crate::symbol::Symbol; +#[cfg(test)] +mod tests; + /// Finds the Levenshtein distance between two strings pub fn lev_distance(a: &str, b: &str) -> usize { // cases which don't require further computation @@ -77,60 +80,3 @@ pub fn find_best_match_for_name<'a, T>(iter_names: T, if let Some((candidate, _)) = levenstein_match { Some(candidate) } else { None } } } - -#[test] -fn test_lev_distance() { - use std::char::{from_u32, MAX}; - // Test bytelength agnosticity - for c in (0..MAX as u32) - .filter_map(|i| from_u32(i)) - .map(|i| i.to_string()) { - assert_eq!(lev_distance(&c[..], &c[..]), 0); - } - - let a = "\nMäry häd ä little lämb\n\nLittle lämb\n"; - let b = "\nMary häd ä little lämb\n\nLittle lämb\n"; - let c = "Mary häd ä little lämb\n\nLittle lämb\n"; - assert_eq!(lev_distance(a, b), 1); - assert_eq!(lev_distance(b, a), 1); - assert_eq!(lev_distance(a, c), 2); - assert_eq!(lev_distance(c, a), 2); - assert_eq!(lev_distance(b, c), 1); - assert_eq!(lev_distance(c, b), 1); -} - -#[test] -fn test_find_best_match_for_name() { - use crate::with_default_globals; - with_default_globals(|| { - let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")]; - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", None), - Some(Symbol::intern("aaab")) - ); - - assert_eq!( - find_best_match_for_name(input.iter(), "1111111111", None), - None - ); - - let input = vec![Symbol::intern("aAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), "AAAA", None), - Some(Symbol::intern("aAAA")) - ); - - let input = vec![Symbol::intern("AAAA")]; - // Returns None because `lev_distance > max_dist / 3` - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", None), - None - ); - - let input = vec![Symbol::intern("AAAA")]; - assert_eq!( - find_best_match_for_name(input.iter(), "aaaa", Some(4)), - Some(Symbol::intern("AAAA")) - ); - }) -} diff --git a/src/libsyntax/util/lev_distance/tests.rs b/src/libsyntax/util/lev_distance/tests.rs new file mode 100644 index 00000000000..1a746a67ec0 --- /dev/null +++ b/src/libsyntax/util/lev_distance/tests.rs @@ -0,0 +1,58 @@ +use super::*; + +#[test] +fn test_lev_distance() { + use std::char::{from_u32, MAX}; + // Test bytelength agnosticity + for c in (0..MAX as u32) + .filter_map(|i| from_u32(i)) + .map(|i| i.to_string()) { + assert_eq!(lev_distance(&c[..], &c[..]), 0); + } + + let a = "\nMäry häd ä little lämb\n\nLittle lämb\n"; + let b = "\nMary häd ä little lämb\n\nLittle lämb\n"; + let c = "Mary häd ä little lämb\n\nLittle lämb\n"; + assert_eq!(lev_distance(a, b), 1); + assert_eq!(lev_distance(b, a), 1); + assert_eq!(lev_distance(a, c), 2); + assert_eq!(lev_distance(c, a), 2); + assert_eq!(lev_distance(b, c), 1); + assert_eq!(lev_distance(c, b), 1); +} + +#[test] +fn test_find_best_match_for_name() { + use crate::with_default_globals; + with_default_globals(|| { + let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")]; + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", None), + Some(Symbol::intern("aaab")) + ); + + assert_eq!( + find_best_match_for_name(input.iter(), "1111111111", None), + None + ); + + let input = vec![Symbol::intern("aAAA")]; + assert_eq!( + find_best_match_for_name(input.iter(), "AAAA", None), + Some(Symbol::intern("aAAA")) + ); + + let input = vec![Symbol::intern("AAAA")]; + // Returns None because `lev_distance > max_dist / 3` + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", None), + None + ); + + let input = vec![Symbol::intern("AAAA")]; + assert_eq!( + find_best_match_for_name(input.iter(), "aaaa", Some(4)), + Some(Symbol::intern("AAAA")) + ); + }) +} diff --git a/src/libsyntax/util/parser_testing.rs b/src/libsyntax/util/parser_testing.rs deleted file mode 100644 index 627422df1db..00000000000 --- a/src/libsyntax/util/parser_testing.rs +++ /dev/null @@ -1,160 +0,0 @@ -use crate::ast::{self, Ident}; -use crate::source_map::FilePathMapping; -use crate::parse::{ParseSess, PResult, source_file_to_stream}; -use crate::parse::new_parser_from_source_str; -use crate::parse::parser::Parser; -use crate::ptr::P; -use crate::tokenstream::TokenStream; - -use std::iter::Peekable; -use std::path::PathBuf; - -/// Map a string to tts, using a made-up filename: -pub fn string_to_stream(source_str: String) -> TokenStream { - let ps = ParseSess::new(FilePathMapping::empty()); - source_file_to_stream( - &ps, - ps.source_map().new_source_file(PathBuf::from("bogofile").into(), - source_str, - ), None).0 -} - -/// Map string to parser (via tts) -pub fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> { - new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str) -} - -fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where - F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>, -{ - let mut p = string_to_parser(&ps, s); - let x = panictry!(f(&mut p)); - p.sess.span_diagnostic.abort_if_errors(); - x -} - -/// Parse a string, return a crate. -pub fn string_to_crate(source_str : String) -> ast::Crate { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_crate_mod() - }) -} - -/// Parse a string, return an expr -pub fn string_to_expr(source_str : String) -> P<ast::Expr> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_expr() - }) -} - -/// Parse a string, return an item -pub fn string_to_item(source_str : String) -> Option<P<ast::Item>> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_item() - }) -} - -/// Parse a string, return a pat. Uses "irrefutable"... which doesn't -/// (currently) affect parsing. -pub fn string_to_pat(source_str: String) -> P<ast::Pat> { - let ps = ParseSess::new(FilePathMapping::empty()); - with_error_checking_parse(source_str, &ps, |p| { - p.parse_pat(None) - }) -} - -/// Converts a vector of strings to a vector of Ident's -pub fn strs_to_idents(ids: Vec<&str> ) -> Vec<Ident> { - ids.iter().map(|u| Ident::from_str(*u)).collect() -} - -/// Does the given string match the pattern? whitespace in the first string -/// may be deleted or replaced with other whitespace to match the pattern. -/// This function is relatively Unicode-ignorant; fortunately, the careful design -/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?). -pub fn matches_codepattern(a : &str, b : &str) -> bool { - let mut a_iter = a.chars().peekable(); - let mut b_iter = b.chars().peekable(); - - loop { - let (a, b) = match (a_iter.peek(), b_iter.peek()) { - (None, None) => return true, - (None, _) => return false, - (Some(&a), None) => { - if is_pattern_whitespace(a) { - break // trailing whitespace check is out of loop for borrowck - } else { - return false - } - } - (Some(&a), Some(&b)) => (a, b) - }; - - if is_pattern_whitespace(a) && is_pattern_whitespace(b) { - // skip whitespace for a and b - scan_for_non_ws_or_end(&mut a_iter); - scan_for_non_ws_or_end(&mut b_iter); - } else if is_pattern_whitespace(a) { - // skip whitespace for a - scan_for_non_ws_or_end(&mut a_iter); - } else if a == b { - a_iter.next(); - b_iter.next(); - } else { - return false - } - } - - // check if a has *only* trailing whitespace - a_iter.all(is_pattern_whitespace) -} - -/// Advances the given peekable `Iterator` until it reaches a non-whitespace character -fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) { - while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) { - iter.next(); - } -} - -pub fn is_pattern_whitespace(c: char) -> bool { - rustc_lexer::character_properties::is_whitespace(c) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn eqmodws() { - assert_eq!(matches_codepattern("",""),true); - assert_eq!(matches_codepattern("","a"),false); - assert_eq!(matches_codepattern("a",""),false); - assert_eq!(matches_codepattern("a","a"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \n\t\r b "),false); - assert_eq!(matches_codepattern("a b","a b"),true); - assert_eq!(matches_codepattern("ab","a b"),false); - assert_eq!(matches_codepattern("a b","ab"),true); - assert_eq!(matches_codepattern(" a b","ab"),true); - } - - #[test] - fn pattern_whitespace() { - assert_eq!(matches_codepattern("","\x0C"), false); - assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true); - assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false); - } - - #[test] - fn non_pattern_whitespace() { - // These have the property 'White_Space' but not 'Pattern_White_Space' - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("a b","a\u{2002}b"), false); - assert_eq!(matches_codepattern("\u{205F}a b","ab"), false); - assert_eq!(matches_codepattern("a \u{3000}b","ab"), false); - } -} |