From b5e35b128efeed4bfdb4b1ee9d0697389ec9f164 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 30 Jul 2019 12:33:32 +0300 Subject: remove special code path for unknown tokens --- src/libsyntax/parse/lexer/mod.rs | 73 +++++++--------------------------------- 1 file changed, 13 insertions(+), 60 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 950b1b2ff53..c209ae1cb9f 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -3,7 +3,7 @@ use crate::parse::token::{self, Token, TokenKind}; use crate::symbol::{sym, Symbol}; use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; -use errors::{FatalError, Diagnostic, DiagnosticBuilder}; +use errors::{FatalError, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION}; use rustc_lexer::Base; use rustc_lexer::unescape; @@ -39,7 +39,6 @@ pub struct StringReader<'a> { pos: BytePos, /// Stop reading src at this index. end_src_index: usize, - fatal_errs: Vec>, /// Source text to tokenize. src: Lrc, override_span: Option, @@ -62,7 +61,6 @@ impl<'a> StringReader<'a> { pos: source_file.start_pos, end_src_index: src.len(), src, - fatal_errs: Vec::new(), override_span, } } @@ -89,29 +87,17 @@ impl<'a> StringReader<'a> { self.override_span.unwrap_or_else(|| Span::new(lo, hi, NO_EXPANSION)) } - fn unwrap_or_abort(&mut self, res: Result) -> Token { - match res { - Ok(tok) => tok, - Err(_) => { - self.emit_fatal_errors(); - FatalError.raise(); - } - } - } - /// Returns the next token, including trivia like whitespace or comments. /// /// `Err(())` means that some errors were encountered, which can be /// retrieved using `buffer_fatal_errors`. - pub fn try_next_token(&mut self) -> Result { - assert!(self.fatal_errs.is_empty()); - + pub fn next_token(&mut self) -> Token { let start_src_index = self.src_index(self.pos); let text: &str = &self.src[start_src_index..self.end_src_index]; if text.is_empty() { let span = self.mk_sp(self.pos, self.pos); - return Ok(Token::new(token::Eof, span)); + return Token::new(token::Eof, span); } { @@ -125,7 +111,7 @@ impl<'a> StringReader<'a> { let kind = token::Shebang(sym); let span = self.mk_sp(start, self.pos); - return Ok(Token::new(kind, span)); + return Token::new(kind, span); } } } @@ -139,39 +125,10 @@ impl<'a> StringReader<'a> { // This could use `?`, but that makes code significantly (10-20%) slower. // https://github.com/rust-lang/rust/issues/37939 - let kind = match self.cook_lexer_token(token.kind, start) { - Ok(it) => it, - Err(err) => return Err(self.fatal_errs.push(err)), - }; + let kind = self.cook_lexer_token(token.kind, start); let span = self.mk_sp(start, self.pos); - Ok(Token::new(kind, span)) - } - - /// Returns the next token, including trivia like whitespace or comments. - /// - /// Aborts in case of an error. - pub fn next_token(&mut self) -> Token { - let res = self.try_next_token(); - self.unwrap_or_abort(res) - } - - fn emit_fatal_errors(&mut self) { - for err in &mut self.fatal_errs { - err.emit(); - } - - self.fatal_errs.clear(); - } - - pub fn buffer_fatal_errors(&mut self) -> Vec { - let mut buffer = Vec::new(); - - for err in self.fatal_errs.drain(..) { - err.buffer(&mut buffer); - } - - buffer + Token::new(kind, span) } /// Report a fatal lexical error with a given span. @@ -218,8 +175,8 @@ impl<'a> StringReader<'a> { &self, token: rustc_lexer::TokenKind, start: BytePos, - ) -> Result> { - let kind = match token { + ) -> TokenKind { + match token { rustc_lexer::TokenKind::LineComment => { let string = self.str_from(start); // comments with only more "/"s are not doc comments @@ -396,16 +353,12 @@ impl<'a> StringReader<'a> { // this should be inside `rustc_lexer`. However, we should first remove compound // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, // as there will be less overall work to do this way. - return match unicode_chars::check_for_substitution(self, start, c, &mut err) { - Some(token) => { - err.emit(); - Ok(token) - } - None => Err(err), - } + let token = unicode_chars::check_for_substitution(self, start, c, &mut err) + .unwrap_or(token::Whitespace); + err.emit(); + token } - }; - Ok(kind) + } } fn cook_lexer_literal( -- cgit 1.4.1-3-g733a5 From 58ac81a60fe11868b0748a406d8e0b97efa4e8c5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 30 Jul 2019 12:31:41 +0300 Subject: add unknown token --- src/librustc/ich/impls_syntax.rs | 3 ++- src/librustdoc/html/highlight.rs | 2 +- src/libsyntax/ext/proc_macro_server.rs | 2 +- src/libsyntax/parse/lexer/mod.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 2 +- src/libsyntax/parse/token.rs | 4 +++- src/libsyntax/print/pprust.rs | 1 + 7 files changed, 10 insertions(+), 6 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 0c9c9adcf9d..5cc8324b316 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -363,7 +363,8 @@ impl<'a> HashStable> for token::TokenKind { } token::DocComment(val) | - token::Shebang(val) => val.hash_stable(hcx, hasher), + token::Shebang(val) | + token::Unknown(val) => val.hash_stable(hcx, hasher), } } } diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 8132074d6e0..92d85ef9cac 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -237,7 +237,7 @@ impl<'a> Classifier<'a> { return Ok(()); }, - token::Whitespace => Class::None, + token::Whitespace | token::Unknown(..) => Class::None, token::Comment => Class::Comment, token::DocComment(..) => Class::DocComment, diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 8d0023c9ab1..36621ce7775 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -184,7 +184,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> } OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Eof => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), } } } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index c209ae1cb9f..e86d4c7fde6 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -354,7 +354,7 @@ impl<'a> StringReader<'a> { // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, // as there will be less overall work to do this way. let token = unicode_chars::check_for_substitution(self, start, c, &mut err) - .unwrap_or(token::Whitespace); + .unwrap_or_else(|| token::Unknown(self.symbol_from(start))); err.emit(); token } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 830fbec58de..37e67a2729e 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -217,7 +217,7 @@ impl<'a> TokenTreesReader<'a> { loop { let token = self.string_reader.next_token(); match token.kind { - token::Whitespace | token::Comment | token::Shebang(_) => { + token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => { self.joint_to_prev = NonJoint; } _ => { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 73adb5c947c..be800b4de66 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -255,6 +255,8 @@ pub enum TokenKind { /// A comment. Comment, Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), Eof, } @@ -603,7 +605,7 @@ impl Token { DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Eof => return None, + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2ef8a919b9c..378ba1e4107 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -288,6 +288,7 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option) token::Whitespace => " ".to_string(), token::Comment => "/* */".to_string(), token::Shebang(s) => format!("/* shebang: {}*/", s), + token::Unknown(s) => s.to_string(), token::Interpolated(ref nt) => nonterminal_to_string(nt), } -- cgit 1.4.1-3-g733a5 From ab3fb1e775ff20f80096953080dfc650950041b0 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 4 Aug 2019 17:59:06 -0400 Subject: Drop span argument from mk_list_item --- src/librustc/hir/lowering.rs | 7 +++---- src/libsyntax/attr/mod.rs | 4 ++-- src/libsyntax/ext/build.rs | 2 +- src/libsyntax/ext/expand.rs | 4 ++-- src/libsyntax/print/pprust.rs | 5 ++--- src/libsyntax_ext/test_harness.rs | 3 +-- 6 files changed, 11 insertions(+), 14 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 5031e6dbd87..493083c680a 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -5176,11 +5176,10 @@ impl<'a> LoweringContext<'a> { let attr = { // `allow(unreachable_code)` let allow = { - let allow_ident = Ident::with_empty_ctxt(sym::allow).with_span_pos(e.span); - let uc_ident = Ident::with_empty_ctxt(sym::unreachable_code) - .with_span_pos(e.span); + let allow_ident = Ident::new(sym::allow, e.span); + let uc_ident = Ident::new(sym::unreachable_code, e.span); let uc_nested = attr::mk_nested_word_item(uc_ident); - attr::mk_list_item(e.span, allow_ident, vec![uc_nested]) + attr::mk_list_item(allow_ident, vec![uc_nested]) }; attr::mk_attr_outer(allow) }; diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 3e56136b171..7eea1aad8ea 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -355,8 +355,8 @@ pub fn mk_name_value_item(span: Span, ident: Ident, lit_kind: LitKind, lit_span: MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(lit) } } -pub fn mk_list_item(span: Span, ident: Ident, items: Vec) -> MetaItem { - MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::List(items) } +pub fn mk_list_item(ident: Ident, items: Vec) -> MetaItem { + MetaItem { path: Path::from_ident(ident), span: ident.span, node: MetaItemKind::List(items) } } pub fn mk_word_item(ident: Ident) -> MetaItem { diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index 59e13fdc8f1..83d95216850 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -879,7 +879,7 @@ impl<'a> ExtCtxt<'a> { pub fn meta_list(&self, sp: Span, name: ast::Name, mis: Vec) -> ast::MetaItem { - attr::mk_list_item(sp, Ident::new(name, sp), mis) + attr::mk_list_item(Ident::new(name, sp), mis) } pub fn meta_name_value(&self, span: Span, name: ast::Name, lit_kind: ast::LitKind) diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index d2807e4a4b5..964c81dd466 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1272,7 +1272,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { ]; let include_ident = Ident::with_empty_ctxt(sym::include); - let item = attr::mk_list_item(DUMMY_SP, include_ident, include_info); + let item = attr::mk_list_item(include_ident, include_info); items.push(ast::NestedMetaItem::MetaItem(item)); } Err(e) => { @@ -1333,7 +1333,7 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { } } - let meta = attr::mk_list_item(DUMMY_SP, Ident::with_empty_ctxt(sym::doc), items); + let meta = attr::mk_list_item(Ident::with_empty_ctxt(sym::doc), items); *at = attr::Attribute { span: at.span, id: at.id, diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2ef8a919b9c..80342687733 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -15,7 +15,7 @@ use crate::tokenstream::{self, TokenStream, TokenTree}; use rustc_target::spec::abi::{self, Abi}; use syntax_pos::{self, BytePos}; -use syntax_pos::{DUMMY_SP, FileName, Span}; +use syntax_pos::{FileName, Span}; use std::borrow::Cow; @@ -124,8 +124,7 @@ pub fn print_crate<'a>(cm: &'a SourceMap, // #![feature(prelude_import)] let pi_nested = attr::mk_nested_word_item(ast::Ident::with_empty_ctxt(sym::prelude_import)); - let list = attr::mk_list_item( - DUMMY_SP, ast::Ident::with_empty_ctxt(sym::feature), vec![pi_nested]); + let list = attr::mk_list_item(ast::Ident::with_empty_ctxt(sym::feature), vec![pi_nested]); let fake_attr = attr::mk_attr_inner(list); s.print_attribute(&fake_attr); diff --git a/src/libsyntax_ext/test_harness.rs b/src/libsyntax_ext/test_harness.rs index 4b3903c7ad7..eec8a3f8023 100644 --- a/src/libsyntax_ext/test_harness.rs +++ b/src/libsyntax_ext/test_harness.rs @@ -157,8 +157,7 @@ impl MutVisitor for EntryPointCleaner { item.map(|ast::Item {id, ident, attrs, node, vis, span, tokens}| { let allow_ident = Ident::with_empty_ctxt(sym::allow); let dc_nested = attr::mk_nested_word_item(Ident::from_str("dead_code")); - let allow_dead_code_item = attr::mk_list_item(DUMMY_SP, allow_ident, - vec![dc_nested]); + let allow_dead_code_item = attr::mk_list_item(allow_ident, vec![dc_nested]); let allow_dead_code = attr::mk_attr_outer(allow_dead_code_item); ast::Item { -- cgit 1.4.1-3-g733a5 From 24a491f40c891e06dce930081d5e5228232f7a18 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 4 Aug 2019 18:03:34 -0400 Subject: Drop explicit span argument from mk_name_value_item --- src/librustdoc/clean/cfg/tests.rs | 1 - src/libsyntax/attr/mod.rs | 5 +++-- src/libsyntax/ext/build.rs | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'src/libsyntax') diff --git a/src/librustdoc/clean/cfg/tests.rs b/src/librustdoc/clean/cfg/tests.rs index d0df9f8f7e4..405144b444f 100644 --- a/src/librustdoc/clean/cfg/tests.rs +++ b/src/librustdoc/clean/cfg/tests.rs @@ -211,7 +211,6 @@ fn test_parse_ok() { fn test_parse_err() { with_default_globals(|| { let mi = attr::mk_name_value_item( - DUMMY_SP, Ident::from_str("foo"), LitKind::Bool(false), DUMMY_SP, diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7eea1aad8ea..28391aa0dec 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -347,11 +347,12 @@ impl Attribute { pub fn mk_name_value_item_str(ident: Ident, value: Spanned) -> MetaItem { let lit_kind = LitKind::Str(value.node, ast::StrStyle::Cooked); - mk_name_value_item(ident.span.to(value.span), ident, lit_kind, value.span) + mk_name_value_item(ident, lit_kind, value.span) } -pub fn mk_name_value_item(span: Span, ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem { +pub fn mk_name_value_item(ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem { let lit = Lit::from_lit_kind(lit_kind, lit_span); + let span = ident.span.to(lit_span); MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(lit) } } diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs index 83d95216850..db562840e8d 100644 --- a/src/libsyntax/ext/build.rs +++ b/src/libsyntax/ext/build.rs @@ -884,8 +884,7 @@ impl<'a> ExtCtxt<'a> { pub fn meta_name_value(&self, span: Span, name: ast::Name, lit_kind: ast::LitKind) -> ast::MetaItem { - attr::mk_name_value_item(span, Ident::new(name, span), - lit_kind, span) + attr::mk_name_value_item(Ident::new(name, span), lit_kind, span) } pub fn item_use(&self, sp: Span, -- cgit 1.4.1-3-g733a5 From 88491497456e485b0171d6791bd81f52170fbc09 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 4 Aug 2019 19:35:29 -0400 Subject: Make mk_attr_id private to libsyntax --- src/libsyntax/attr/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 28391aa0dec..a9d3227b3a8 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -368,7 +368,7 @@ pub fn mk_nested_word_item(ident: Ident) -> NestedMetaItem { NestedMetaItem::MetaItem(mk_word_item(ident)) } -pub fn mk_attr_id() -> AttrId { +crate fn mk_attr_id() -> AttrId { use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -- cgit 1.4.1-3-g733a5 From fbf93d4931ac9878943cb644d9c1b993b1d20a80 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Mon, 5 Aug 2019 09:14:51 -0400 Subject: Remove leftover AwaitOrigin This was missed in PR #62293. --- src/libsyntax/ast.rs | 9 --------- 1 file changed, 9 deletions(-) (limited to 'src/libsyntax') diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 849e77c4f31..052eb55b408 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -1284,15 +1284,6 @@ pub enum Movability { Movable, } -/// Whether an `await` comes from `await!` or `.await` syntax. -/// FIXME: this should be removed when support for legacy `await!` is removed. -/// https://github.com/rust-lang/rust/issues/60610 -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)] -pub enum AwaitOrigin { - FieldLike, - MacroLike, -} - pub type Mac = Spanned; /// Represents a macro invocation. The `Path` indicates which macro -- cgit 1.4.1-3-g733a5