diff options
28 files changed, 244 insertions, 171 deletions
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 74a0c13b23f..2e86970bcfd 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -1734,8 +1734,10 @@ pub enum StrStyle { /// A literal in a meta item. #[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)] pub struct MetaItemLit { - /// The original literal token as written in source code. - pub token_lit: token::Lit, + /// The original literal as written in the source code. + pub symbol: Symbol, + /// The original suffix as written in the source code. + pub suffix: Option<Symbol>, /// The "semantic" representation of the literal lowered from the original tokens. /// Strings are unescaped, hexadecimal forms are eliminated, etc. pub kind: LitKind, @@ -1745,13 +1747,14 @@ pub struct MetaItemLit { /// Similar to `MetaItemLit`, but restricted to string literals. #[derive(Clone, Copy, Encodable, Decodable, Debug)] pub struct StrLit { - /// The original literal token as written in source code. - pub style: StrStyle, + /// The original literal as written in source code. pub symbol: Symbol, + /// The original suffix as written in source code. pub suffix: Option<Symbol>, - pub span: Span, - /// The unescaped "semantic" representation of the literal lowered from the original token. + /// The semantic (unescaped) representation of the literal. pub symbol_unescaped: Symbol, + pub style: StrStyle, + pub span: Span, } impl StrLit { @@ -1797,8 +1800,9 @@ pub enum LitKind { /// A string literal (`"foo"`). The symbol is unescaped, and so may differ /// from the original token's symbol. Str(Symbol, StrStyle), - /// A byte string (`b"foo"`). - ByteStr(Lrc<[u8]>), + /// A byte string (`b"foo"`). Not stored as a symbol because it might be + /// non-utf8, and symbols only allow utf8 strings. + ByteStr(Lrc<[u8]>, StrStyle), /// A byte char (`b'f'`). Byte(u8), /// A character literal (`'a'`). @@ -1823,7 +1827,7 @@ impl LitKind { /// Returns `true` if this literal is byte literal string. pub fn is_bytestr(&self) -> bool { - matches!(self, LitKind::ByteStr(_)) + matches!(self, LitKind::ByteStr(..)) } /// Returns `true` if this is a numeric literal. @@ -3100,7 +3104,7 @@ mod size_asserts { static_assert_size!(ItemKind, 112); static_assert_size!(LitKind, 24); static_assert_size!(Local, 72); - static_assert_size!(MetaItemLit, 48); + static_assert_size!(MetaItemLit, 40); static_assert_size!(Param, 40); static_assert_size!(Pat, 88); static_assert_size!(Path, 24); diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 057cc26b579..d99f6ed2c1c 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -1,15 +1,15 @@ //! Functions dealing with attributes and meta items. -use crate::ast; use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute}; use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit}; use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr}; -use crate::ast::{Path, PathSegment, StrStyle, DUMMY_NODE_ID}; +use crate::ast::{Path, PathSegment, DUMMY_NODE_ID}; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, Token}; use crate::tokenstream::{DelimSpan, Spacing, TokenTree}; use crate::tokenstream::{LazyAttrTokenStream, TokenStream}; use crate::util::comments; +use crate::util::literal::escape_string_symbol; use rustc_data_structures::sync::WorkerLocal; use rustc_index::bit_set::GrowableBitSet; use rustc_span::symbol::{sym, Ident, Symbol}; @@ -321,18 +321,6 @@ impl Attribute { } } -/* Constructors */ - -pub fn mk_name_value_item_str(ident: Ident, str: Symbol, str_span: Span) -> MetaItem { - mk_name_value_item(ident, LitKind::Str(str, ast::StrStyle::Cooked), str_span) -} - -pub fn mk_name_value_item(ident: Ident, kind: LitKind, lit_span: Span) -> MetaItem { - let lit = MetaItemLit { token_lit: kind.to_token_lit(), kind, span: lit_span }; - let span = ident.span.to(lit_span); - MetaItem { path: Path::from_ident(ident), kind: MetaItemKind::NameValue(lit), span } -} - pub struct AttrIdGenerator(WorkerLocal<Cell<u32>>); #[cfg(debug_assertions)] @@ -408,7 +396,7 @@ pub fn mk_attr_name_value_str( val: Symbol, span: Span, ) -> Attribute { - let lit = LitKind::Str(val, StrStyle::Cooked).to_token_lit(); + let lit = token::Lit::new(token::Str, escape_string_symbol(val), None); let expr = P(Expr { id: DUMMY_NODE_ID, kind: ExprKind::Lit(lit), diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs index f6f186b5107..0daeecb53a8 100644 --- a/compiler/rustc_ast/src/util/literal.rs +++ b/compiler/rustc_ast/src/util/literal.rs @@ -1,11 +1,31 @@ //! Code related to parsing literals. -use crate::ast::{self, LitKind, MetaItemLit}; +use crate::ast::{self, LitKind, MetaItemLit, StrStyle}; use crate::token::{self, Token}; use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode}; use rustc_span::symbol::{kw, sym, Symbol}; use rustc_span::Span; -use std::ascii; +use std::{ascii, fmt, str}; + +// Escapes a string, represented as a symbol. Reuses the original symbol, +// avoiding interning, if no changes are required. +pub fn escape_string_symbol(symbol: Symbol) -> Symbol { + let s = symbol.as_str(); + let escaped = s.escape_default().to_string(); + if s == escaped { symbol } else { Symbol::intern(&escaped) } +} + +// Escapes a char. +pub fn escape_char_symbol(ch: char) -> Symbol { + let s: String = ch.escape_default().map(Into::<char>::into).collect(); + Symbol::intern(&s) +} + +// Escapes a byte string. +pub fn escape_byte_str_symbol(bytes: &[u8]) -> Symbol { + let s = bytes.escape_ascii().to_string(); + Symbol::intern(&s) +} #[derive(Debug)] pub enum LitError { @@ -115,9 +135,9 @@ impl LitKind { } }); error?; - LitKind::ByteStr(buf.into()) + LitKind::ByteStr(buf.into(), StrStyle::Cooked) } - token::ByteStrRaw(_) => { + token::ByteStrRaw(n) => { let s = symbol.as_str(); let bytes = if s.contains('\r') { let mut buf = Vec::with_capacity(s.len()); @@ -136,69 +156,95 @@ impl LitKind { symbol.to_string().into_bytes() }; - LitKind::ByteStr(bytes.into()) + LitKind::ByteStr(bytes.into(), StrStyle::Raw(n)) } token::Err => LitKind::Err, }) } +} - /// Attempts to recover a token from semantic literal. - /// This function is used when the original token doesn't exist (e.g. the literal is created - /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing). - pub fn to_token_lit(&self) -> token::Lit { - let (kind, symbol, suffix) = match *self { - LitKind::Str(symbol, ast::StrStyle::Cooked) => { - // Don't re-intern unless the escaped string is different. - let s = symbol.as_str(); - let escaped = s.escape_default().to_string(); - let symbol = if s == escaped { symbol } else { Symbol::intern(&escaped) }; - (token::Str, symbol, None) - } - LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None), - LitKind::ByteStr(ref bytes) => { - let string = bytes.escape_ascii().to_string(); - (token::ByteStr, Symbol::intern(&string), None) +impl fmt::Display for LitKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + LitKind::Byte(b) => { + let b: String = ascii::escape_default(b).map(Into::<char>::into).collect(); + write!(f, "b'{}'", b)?; } - LitKind::Byte(byte) => { - let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect(); - (token::Byte, Symbol::intern(&string), None) + LitKind::Char(ch) => write!(f, "'{}'", escape_char_symbol(ch))?, + LitKind::Str(sym, StrStyle::Cooked) => write!(f, "\"{}\"", escape_string_symbol(sym))?, + LitKind::Str(sym, StrStyle::Raw(n)) => write!( + f, + "r{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = sym + )?, + LitKind::ByteStr(ref bytes, StrStyle::Cooked) => { + write!(f, "b\"{}\"", escape_byte_str_symbol(bytes))? } - LitKind::Char(ch) => { - let string: String = ch.escape_default().map(Into::<char>::into).collect(); - (token::Char, Symbol::intern(&string), None) + LitKind::ByteStr(ref bytes, StrStyle::Raw(n)) => { + // Unwrap because raw byte string literals can only contain ASCII. + let symbol = str::from_utf8(bytes).unwrap(); + write!( + f, + "br{delim}\"{string}\"{delim}", + delim = "#".repeat(n as usize), + string = symbol + )?; } LitKind::Int(n, ty) => { - let suffix = match ty { - ast::LitIntType::Unsigned(ty) => Some(ty.name()), - ast::LitIntType::Signed(ty) => Some(ty.name()), - ast::LitIntType::Unsuffixed => None, - }; - (token::Integer, sym::integer(n), suffix) + write!(f, "{}", n)?; + match ty { + ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?, + ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?, + ast::LitIntType::Unsuffixed => {} + } } LitKind::Float(symbol, ty) => { - let suffix = match ty { - ast::LitFloatType::Suffixed(ty) => Some(ty.name()), - ast::LitFloatType::Unsuffixed => None, - }; - (token::Float, symbol, suffix) + write!(f, "{}", symbol)?; + match ty { + ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?, + ast::LitFloatType::Unsuffixed => {} + } } - LitKind::Bool(value) => { - let symbol = if value { kw::True } else { kw::False }; - (token::Bool, symbol, None) + LitKind::Bool(b) => write!(f, "{}", if b { "true" } else { "false" })?, + LitKind::Err => { + // This only shows up in places like `-Zunpretty=hir` output, so we + // don't bother to produce something useful. + write!(f, "<bad-literal>")?; } - // This only shows up in places like `-Zunpretty=hir` output, so we - // don't bother to produce something useful. - LitKind::Err => (token::Err, Symbol::intern("<bad-literal>"), None), - }; + } - token::Lit::new(kind, symbol, suffix) + Ok(()) } } impl MetaItemLit { - /// Converts token literal into a meta item literal. + /// Converts a token literal into a meta item literal. pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> { - Ok(MetaItemLit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span }) + Ok(MetaItemLit { + symbol: token_lit.symbol, + suffix: token_lit.suffix, + kind: LitKind::from_token_lit(token_lit)?, + span, + }) + } + + /// Cheaply converts a meta item literal into a token literal. + pub fn as_token_lit(&self) -> token::Lit { + let kind = match self.kind { + LitKind::Bool(_) => token::Bool, + LitKind::Str(_, ast::StrStyle::Cooked) => token::Str, + LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n), + LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr, + LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n), + LitKind::Byte(_) => token::Byte, + LitKind::Char(_) => token::Char, + LitKind::Int(..) => token::Integer, + LitKind::Float(..) => token::Float, + LitKind::Err => token::Err, + }; + + token::Lit::new(kind, self.symbol, self.suffix) } /// Converts an arbitrary token into meta item literal. diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 24e2985cf56..3946aab646a 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -97,7 +97,7 @@ impl<'hir> LoweringContext<'_, 'hir> { } ExprKind::IncludedBytes(bytes) => hir::ExprKind::Lit(respan( self.lower_span(e.span), - LitKind::ByteStr(bytes.clone()), + LitKind::ByteStr(bytes.clone(), StrStyle::Cooked), )), ExprKind::Cast(expr, ty) => { let expr = self.lower_expr(expr); diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 5d7397977e9..d67ede6e130 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -958,7 +958,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { lit } else { MetaItemLit { - token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None), + symbol: kw::Empty, + suffix: None, kind: LitKind::Err, span: DUMMY_SP, } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index d3d8431c163..d0e4761a10d 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -376,7 +376,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere } fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) { - self.print_token_literal(lit.token_lit, lit.span) + self.print_token_literal(lit.as_token_lit(), lit.span) } fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) { diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs index 4ed16e337d2..3b17f6dd627 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs @@ -2,6 +2,8 @@ use crate::pp::Breaks::Inconsistent; use crate::pprust::state::{AnnNode, IterDelimited, PrintState, State, INDENT_UNIT}; use rustc_ast::ptr::P; +use rustc_ast::token; +use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast::util::parser::{self, AssocOp, Fixity}; use rustc_ast::{self as ast, BlockCheckMode}; @@ -323,7 +325,7 @@ impl<'a> State<'a> { self.print_token_literal(*token_lit, expr.span); } ast::ExprKind::IncludedBytes(bytes) => { - let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit(); + let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None); self.print_token_literal(lit, expr.span) } ast::ExprKind::Cast(expr, ty) => { diff --git a/compiler/rustc_builtin_macros/src/concat_bytes.rs b/compiler/rustc_builtin_macros/src/concat_bytes.rs index 70ce5a6c419..4c424ea430b 100644 --- a/compiler/rustc_builtin_macros/src/concat_bytes.rs +++ b/compiler/rustc_builtin_macros/src/concat_bytes.rs @@ -69,7 +69,7 @@ fn invalid_type_err( Ok(ast::LitKind::Int(_, _)) => { cx.span_err(span, "numeric literal is not a `u8`"); } - Ok(ast::LitKind::ByteStr(_) | ast::LitKind::Byte(_)) => unreachable!(), + Ok(ast::LitKind::ByteStr(..) | ast::LitKind::Byte(_)) => unreachable!(), Err(err) => { report_lit_error(&cx.sess.parse_sess, err, token_lit, span); } @@ -97,7 +97,7 @@ fn handle_array_element( )) if val <= u8::MAX.into() => Some(val as u8), Ok(ast::LitKind::Byte(val)) => Some(val), - Ok(ast::LitKind::ByteStr(_)) => { + Ok(ast::LitKind::ByteStr(..)) => { if !*has_errors { cx.struct_span_err(expr.span, "cannot concatenate doubly nested array") .note("byte strings are treated as arrays of bytes") @@ -174,7 +174,7 @@ pub fn expand_concat_bytes( Ok(ast::LitKind::Byte(val)) => { accumulator.push(val); } - Ok(ast::LitKind::ByteStr(ref bytes)) => { + Ok(ast::LitKind::ByteStr(ref bytes, _)) => { accumulator.extend_from_slice(&bytes); } _ => { diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs index fa5a45730ac..2a8dc02849e 100644 --- a/compiler/rustc_builtin_macros/src/derive.rs +++ b/compiler/rustc_builtin_macros/src/derive.rs @@ -1,7 +1,7 @@ use crate::cfg_eval::cfg_eval; use rustc_ast as ast; -use rustc_ast::{token, GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind}; +use rustc_ast::{GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind}; use rustc_errors::{struct_span_err, Applicability}; use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier}; use rustc_feature::AttributeTemplate; @@ -130,9 +130,11 @@ fn report_bad_target(sess: &Session, item: &Annotatable, span: Span) -> bool { } fn report_unexpected_meta_item_lit(sess: &Session, lit: &ast::MetaItemLit) { - let help_msg = match lit.token_lit.kind { - token::Str if rustc_lexer::is_ident(lit.token_lit.symbol.as_str()) => { - format!("try using `#[derive({})]`", lit.token_lit.symbol) + let help_msg = match lit.kind { + ast::LitKind::Str(_, ast::StrStyle::Cooked) + if rustc_lexer::is_ident(lit.symbol.as_str()) => + { + format!("try using `#[derive({})]`", lit.symbol) } _ => "for example, write `#[derive(Debug)]` for `Debug`".to_string(), }; diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 86d867f6f27..00453f78287 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -1231,7 +1231,7 @@ pub fn expr_to_spanned_string<'a>( Err(match expr.kind { ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) { Ok(ast::LitKind::Str(s, style)) => return Ok((s, style, expr.span)), - Ok(ast::LitKind::ByteStr(_)) => { + Ok(ast::LitKind::ByteStr(..)) => { let mut err = cx.struct_span_err(expr.span, err_msg); let span = expr.span.shrink_to_lo(); err.span_suggestion( diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index 4812bdd9dd8..ef50efb8125 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -1,8 +1,7 @@ use crate::base::ExtCtxt; -use rustc_ast::attr; use rustc_ast::ptr::P; use rustc_ast::{self as ast, AttrVec, BlockCheckMode, Expr, LocalKind, PatKind, UnOp}; -use rustc_data_structures::sync::Lrc; +use rustc_ast::{attr, token, util::literal}; use rustc_span::source_map::Spanned; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::Span; @@ -332,36 +331,36 @@ impl<'a> ExtCtxt<'a> { self.expr_struct(span, self.path_ident(span, id), fields) } - fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> { - let token_lit = lit_kind.to_token_lit(); - self.expr(span, ast::ExprKind::Lit(token_lit)) + pub fn expr_usize(&self, span: Span, n: usize) -> P<ast::Expr> { + let suffix = Some(ast::UintTy::Usize.name()); + let lit = token::Lit::new(token::Integer, sym::integer(n), suffix); + self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { - self.expr_lit( - span, - ast::LitKind::Int(i as u128, ast::LitIntType::Unsigned(ast::UintTy::Usize)), - ) - } - - pub fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Int(u as u128, ast::LitIntType::Unsigned(ast::UintTy::U32))) + pub fn expr_u32(&self, span: Span, n: u32) -> P<ast::Expr> { + let suffix = Some(ast::UintTy::U32.name()); + let lit = token::Lit::new(token::Integer, sym::integer(n), suffix); + self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Bool(value)) + pub fn expr_bool(&self, span: Span, value: bool) -> P<ast::Expr> { + let lit = token::Lit::new(token::Bool, if value { kw::True } else { kw::False }, None); + self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_str(&self, sp: Span, s: Symbol) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Str(s, ast::StrStyle::Cooked)) + pub fn expr_str(&self, span: Span, s: Symbol) -> P<ast::Expr> { + let lit = token::Lit::new(token::Str, literal::escape_string_symbol(s), None); + self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_char(&self, sp: Span, ch: char) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::Char(ch)) + pub fn expr_char(&self, span: Span, ch: char) -> P<ast::Expr> { + let lit = token::Lit::new(token::Char, literal::escape_char_symbol(ch), None); + self.expr(span, ast::ExprKind::Lit(lit)) } - pub fn expr_byte_str(&self, sp: Span, bytes: Vec<u8>) -> P<ast::Expr> { - self.expr_lit(sp, ast::LitKind::ByteStr(Lrc::from(bytes))) + pub fn expr_byte_str(&self, span: Span, bytes: Vec<u8>) -> P<ast::Expr> { + let lit = token::Lit::new(token::ByteStr, literal::escape_byte_str_symbol(&bytes), None); + self.expr(span, ast::ExprKind::Lit(lit)) } /// `[expr1, expr2, ...]` diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 76165796117..768bdab8a54 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -6,6 +6,7 @@ use pm::{Delimiter, Level, LineColumn}; use rustc_ast as ast; use rustc_ast::token; use rustc_ast::tokenstream::{self, Spacing::*, TokenStream}; +use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast_pretty::pprust; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -526,7 +527,7 @@ impl server::TokenStream for Rustc<'_, '_> { Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span)) } ast::ExprKind::IncludedBytes(bytes) => { - let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit(); + let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None); Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span)) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 94bab9f339d..ef98c4ba54c 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -1245,7 +1245,7 @@ impl<'a> State<'a> { fn print_literal(&mut self, lit: &hir::Lit) { self.maybe_print_comment(lit.span.lo()); - self.word(lit.node.to_token_lit().to_string()) + self.word(lit.node.to_string()) } fn print_inline_asm(&mut self, asm: &hir::InlineAsm<'_>) { diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs index 60fec05d36d..93618c61417 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs @@ -1169,7 +1169,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { match lit.node { ast::LitKind::Str(..) => tcx.mk_static_str(), - ast::LitKind::ByteStr(ref v) => { + ast::LitKind::ByteStr(ref v, _) => { tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.mk_array(tcx.types.u8, v.len() as u64)) } ast::LitKind::Byte(_) => tcx.types.u8, diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs index decd317d9fc..6810353f9e7 100644 --- a/compiler/rustc_hir_typeck/src/pat.rs +++ b/compiler/rustc_hir_typeck/src/pat.rs @@ -386,7 +386,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // Byte string patterns behave the same way as array patterns // They can denote both statically and dynamically-sized byte arrays. let mut pat_ty = ty; - if let hir::ExprKind::Lit(Spanned { node: ast::LitKind::ByteStr(_), .. }) = lt.kind { + if let hir::ExprKind::Lit(Spanned { node: ast::LitKind::ByteStr(..), .. }) = lt.kind { let expected = self.structurally_resolved_type(span, expected); if let ty::Ref(_, inner_ty, _) = expected.kind() && matches!(inner_ty.kind(), ty::Slice(_)) diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs index 717c6231574..3b7ed818dc9 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs @@ -135,14 +135,14 @@ pub(crate) fn lit_to_mir_constant<'tcx>( let allocation = tcx.intern_const_alloc(allocation); ConstValue::Slice { data: allocation, start: 0, end: s.len() } } - (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) + (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Slice(_)) => { let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]); let allocation = tcx.intern_const_alloc(allocation); ConstValue::Slice { data: allocation, start: 0, end: data.len() } } - (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => { + (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => { let id = tcx.allocate_bytes(data); ConstValue::Scalar(Scalar::from_pointer(id.into(), &tcx)) } diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs index a9ed945d4a1..57ae6a3652d 100644 --- a/compiler/rustc_mir_build/src/thir/constant.rs +++ b/compiler/rustc_mir_build/src/thir/constant.rs @@ -33,13 +33,13 @@ pub(crate) fn lit_to_const<'tcx>( let str_bytes = s.as_str().as_bytes(); ty::ValTree::from_raw_bytes(tcx, str_bytes) } - (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) + (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Slice(_)) => { let bytes = data as &[u8]; ty::ValTree::from_raw_bytes(tcx, bytes) } - (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => { + (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => { let bytes = data as &[u8]; ty::ValTree::from_raw_bytes(tcx, bytes) } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index f6a6ed379a2..8a3545eb3be 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1534,15 +1534,16 @@ impl<'a> Parser<'a> { && (matches!(self.token.kind, token::CloseDelim(_) | token::Comma) || self.token.is_op()) { - let lit = self.recover_unclosed_char(label_.ident, |self_| { - self_.sess.create_err(UnexpectedTokenAfterLabel { - span: self_.token.span, - remove_label: None, - enclose_in_block: None, - }) - }); + let (lit, _) = + self.recover_unclosed_char(label_.ident, Parser::mk_token_lit_char, |self_| { + self_.sess.create_err(UnexpectedTokenAfterLabel { + span: self_.token.span, + remove_label: None, + enclose_in_block: None, + }) + }); consume_colon = false; - Ok(self.mk_expr(lo, ExprKind::Lit(lit.token_lit))) + Ok(self.mk_expr(lo, ExprKind::Lit(lit))) } else if !ate_colon && (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt)) { @@ -1617,12 +1618,13 @@ impl<'a> Parser<'a> { Ok(expr) } - /// Emit an error when a char is parsed as a lifetime because of a missing quote - pub(super) fn recover_unclosed_char( + /// Emit an error when a char is parsed as a lifetime because of a missing quote. + pub(super) fn recover_unclosed_char<L>( &self, lifetime: Ident, + mk_lit_char: impl FnOnce(Symbol, Span) -> L, err: impl FnOnce(&Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>, - ) -> ast::MetaItemLit { + ) -> L { if let Some(mut diag) = self.sess.span_diagnostic.steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar) { @@ -1644,11 +1646,7 @@ impl<'a> Parser<'a> { .emit(); } let name = lifetime.without_first_quote().name; - ast::MetaItemLit { - token_lit: token::Lit::new(token::LitKind::Char, name, None), - kind: ast::LitKind::Char(name.as_str().chars().next().unwrap_or('_')), - span: lifetime.span, - } + mk_lit_char(name, lifetime.span) } /// Recover on the syntax `do catch { ... }` suggesting `try { ... }` instead. @@ -1764,8 +1762,8 @@ impl<'a> Parser<'a> { Some(lit) => match lit.kind { ast::LitKind::Str(symbol_unescaped, style) => Ok(ast::StrLit { style, - symbol: lit.token_lit.symbol, - suffix: lit.token_lit.suffix, + symbol: lit.symbol, + suffix: lit.suffix, span: lit.span, symbol_unescaped, }), @@ -1775,7 +1773,23 @@ impl<'a> Parser<'a> { } } - fn handle_missing_lit(&mut self) -> PResult<'a, MetaItemLit> { + pub(crate) fn mk_token_lit_char(name: Symbol, span: Span) -> (token::Lit, Span) { + (token::Lit { symbol: name, suffix: None, kind: token::Char }, span) + } + + fn mk_meta_item_lit_char(name: Symbol, span: Span) -> MetaItemLit { + ast::MetaItemLit { + symbol: name, + suffix: None, + kind: ast::LitKind::Char(name.as_str().chars().next().unwrap_or('_')), + span, + } + } + + fn handle_missing_lit<L>( + &mut self, + mk_lit_char: impl FnOnce(Symbol, Span) -> L, + ) -> PResult<'a, L> { if let token::Interpolated(inner) = &self.token.kind { let expr = match inner.as_ref() { token::NtExpr(expr) => Some(expr), @@ -1799,7 +1813,7 @@ impl<'a> Parser<'a> { // On an error path, eagerly consider a lifetime to be an unclosed character lit if self.token.is_lifetime() { let lt = self.expect_lifetime(); - Ok(self.recover_unclosed_char(lt.ident, err)) + Ok(self.recover_unclosed_char(lt.ident, mk_lit_char, err)) } else { Err(err(self)) } @@ -1808,11 +1822,13 @@ impl<'a> Parser<'a> { pub(super) fn parse_token_lit(&mut self) -> PResult<'a, (token::Lit, Span)> { self.parse_opt_token_lit() .ok_or(()) - .or_else(|()| self.handle_missing_lit().map(|lit| (lit.token_lit, lit.span))) + .or_else(|()| self.handle_missing_lit(Parser::mk_token_lit_char)) } pub(super) fn parse_meta_item_lit(&mut self) -> PResult<'a, MetaItemLit> { - self.parse_opt_meta_item_lit().ok_or(()).or_else(|()| self.handle_missing_lit()) + self.parse_opt_meta_item_lit() + .ok_or(()) + .or_else(|()| self.handle_missing_lit(Parser::mk_meta_item_lit_char)) } fn recover_after_dot(&mut self) -> Option<Token> { diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index cbeec951e2d..a1981e11477 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -411,16 +411,20 @@ impl<'a> Parser<'a> { { // Recover a `'a` as a `'a'` literal let lt = self.expect_lifetime(); - let lit = self.recover_unclosed_char(lt.ident, |self_| { - let expected = expected.unwrap_or("pattern"); - let msg = - format!("expected {}, found {}", expected, super::token_descr(&self_.token)); + let (lit, _) = + self.recover_unclosed_char(lt.ident, Parser::mk_token_lit_char, |self_| { + let expected = expected.unwrap_or("pattern"); + let msg = format!( + "expected {}, found {}", + expected, + super::token_descr(&self_.token) + ); - let mut err = self_.struct_span_err(self_.token.span, &msg); - err.span_label(self_.token.span, format!("expected {}", expected)); - err - }); - PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit.token_lit))) + let mut err = self_.struct_span_err(self_.token.span, &msg); + err.span_label(self_.token.span, format!("expected {}", expected)); + err + }); + PatKind::Lit(self.mk_expr(lo, ExprKind::Lit(lit))) } else { // Try to parse everything else as literal with optional minus match self.parse_literal_maybe_minus() { diff --git a/src/librustdoc/clean/cfg/tests.rs b/src/librustdoc/clean/cfg/tests.rs index 7f72d5d39a7..81f67672436 100644 --- a/src/librustdoc/clean/cfg/tests.rs +++ b/src/librustdoc/clean/cfg/tests.rs @@ -1,9 +1,8 @@ use super::*; -use rustc_ast::attr; -use rustc_ast::Path; +use rustc_ast::{LitKind, MetaItemLit, Path, StrStyle}; use rustc_span::create_default_session_globals_then; -use rustc_span::symbol::{Ident, Symbol}; +use rustc_span::symbol::{kw, Ident, Symbol}; use rustc_span::DUMMY_SP; fn word_cfg(s: &str) -> Cfg { @@ -22,6 +21,15 @@ fn dummy_meta_item_word(name: &str) -> MetaItem { } } +fn dummy_meta_item_name_value(name: &str, symbol: Symbol, kind: LitKind) -> MetaItem { + let lit = MetaItemLit { symbol, suffix: None, kind, span: DUMMY_SP }; + MetaItem { + path: Path::from_ident(Ident::from_str(name)), + kind: MetaItemKind::NameValue(lit), + span: DUMMY_SP, + } +} + macro_rules! dummy_meta_item_list { ($name:ident, [$($list:ident),* $(,)?]) => { MetaItem { @@ -242,8 +250,8 @@ fn test_parse_ok() { let mi = dummy_meta_item_word("all"); assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all"))); - let mi = - attr::mk_name_value_item_str(Ident::from_str("all"), Symbol::intern("done"), DUMMY_SP); + let done = Symbol::intern("done"); + let mi = dummy_meta_item_name_value("all", done, LitKind::Str(done, StrStyle::Cooked)); assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done"))); let mi = dummy_meta_item_list!(all, [a, b]); @@ -272,7 +280,7 @@ fn test_parse_ok() { #[test] fn test_parse_err() { create_default_session_globals_then(|| { - let mi = attr::mk_name_value_item(Ident::from_str("foo"), LitKind::Bool(false), DUMMY_SP); + let mi = dummy_meta_item_name_value("foo", kw::False, LitKind::Bool(false)); assert!(Cfg::parse(&mi).is_err()); let mi = dummy_meta_item_list!(not, [a, b]); diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 2590bb0df3f..98329e7fc91 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -10,7 +10,6 @@ use std::{cmp, fmt, iter}; use arrayvec::ArrayVec; use thin_vec::ThinVec; -use rustc_ast::attr; use rustc_ast::util::comments::beautify_doc_string; use rustc_ast::{self as ast, AttrStyle}; use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel}; @@ -27,7 +26,6 @@ use rustc_middle::ty::fast_reject::SimplifiedType; use rustc_middle::ty::{self, DefIdTree, TyCtxt, Visibility}; use rustc_session::Session; use rustc_span::hygiene::MacroKind; -use rustc_span::source_map::DUMMY_SP; use rustc_span::symbol::{kw, sym, Ident, Symbol}; use rustc_span::{self, FileName, Loc}; use rustc_target::abi::VariantIdx; @@ -982,12 +980,12 @@ impl AttributesExt for [ast::Attribute] { // #[doc(cfg(target_feature = "feat"))] attributes as well for attr in self.lists(sym::target_feature) { if attr.has_name(sym::enable) { - if let Some(feat) = attr.value_str() { - let meta = attr::mk_name_value_item_str( - Ident::with_dummy_span(sym::target_feature), - feat, - DUMMY_SP, - ); + if attr.value_str().is_some() { + // Clone `enable = "feat"`, change to `target_feature = "feat"`. + // Unwrap is safe because `value_str` succeeded above. + let mut meta = attr.meta_item().unwrap().clone(); + meta.path = ast::Path::from_ident(Ident::with_dummy_span(sym::target_feature)); + if let Ok(feat_cfg) = Cfg::parse(&meta) { cfg &= feat_cfg; } diff --git a/src/tools/clippy/clippy_lints/src/invalid_utf8_in_unchecked.rs b/src/tools/clippy/clippy_lints/src/invalid_utf8_in_unchecked.rs index e0a607f9a95..6a4861747d2 100644 --- a/src/tools/clippy/clippy_lints/src/invalid_utf8_in_unchecked.rs +++ b/src/tools/clippy/clippy_lints/src/invalid_utf8_in_unchecked.rs @@ -33,7 +33,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidUtf8InUnchecked { if let Some([arg]) = match_function_call(cx, expr, &paths::STR_FROM_UTF8_UNCHECKED) { match &arg.kind { ExprKind::Lit(Spanned { node: lit, .. }) => { - if let LitKind::ByteStr(bytes) = &lit + if let LitKind::ByteStr(bytes, _) = &lit && std::str::from_utf8(bytes).is_err() { lint(cx, expr.span); diff --git a/src/tools/clippy/clippy_lints/src/large_include_file.rs b/src/tools/clippy/clippy_lints/src/large_include_file.rs index 84dd61a1e4b..424c0d9e798 100644 --- a/src/tools/clippy/clippy_lints/src/large_include_file.rs +++ b/src/tools/clippy/clippy_lints/src/large_include_file.rs @@ -60,7 +60,7 @@ impl LateLintPass<'_> for LargeIncludeFile { then { let len = match &lit.node { // include_bytes - LitKind::ByteStr(bstr) => bstr.len(), + LitKind::ByteStr(bstr, _) => bstr.len(), // include_str LitKind::Str(sym, _) => sym.as_str().len(), _ => return, diff --git a/src/tools/clippy/clippy_lints/src/matches/match_same_arms.rs b/src/tools/clippy/clippy_lints/src/matches/match_same_arms.rs index 168c1e4d2e6..158e6caa4de 100644 --- a/src/tools/clippy/clippy_lints/src/matches/match_same_arms.rs +++ b/src/tools/clippy/clippy_lints/src/matches/match_same_arms.rs @@ -282,7 +282,7 @@ impl<'a> NormalizedPat<'a> { // TODO: Handle negative integers. They're currently treated as a wild match. ExprKind::Lit(lit) => match lit.node { LitKind::Str(sym, _) => Self::LitStr(sym), - LitKind::ByteStr(ref bytes) => Self::LitBytes(bytes), + LitKind::ByteStr(ref bytes, _) => Self::LitBytes(bytes), LitKind::Byte(val) => Self::LitInt(val.into()), LitKind::Char(val) => Self::LitInt(val.into()), LitKind::Int(val, _) => Self::LitInt(val), diff --git a/src/tools/clippy/clippy_lints/src/utils/author.rs b/src/tools/clippy/clippy_lints/src/utils/author.rs index 0c052d86eda..bd7daf0773c 100644 --- a/src/tools/clippy/clippy_lints/src/utils/author.rs +++ b/src/tools/clippy/clippy_lints/src/utils/author.rs @@ -299,7 +299,7 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { }; kind!("Float(_, {float_ty})"); }, - LitKind::ByteStr(ref vec) => { + LitKind::ByteStr(ref vec, _) => { bind!(self, vec); kind!("ByteStr(ref {vec})"); chain!(self, "let [{:?}] = **{vec}", vec.value); diff --git a/src/tools/clippy/clippy_utils/src/check_proc_macro.rs b/src/tools/clippy/clippy_utils/src/check_proc_macro.rs index c6bf98b7b8b..43f0df145f0 100644 --- a/src/tools/clippy/clippy_utils/src/check_proc_macro.rs +++ b/src/tools/clippy/clippy_utils/src/check_proc_macro.rs @@ -69,7 +69,9 @@ fn lit_search_pat(lit: &LitKind) -> (Pat, Pat) { LitKind::Str(_, StrStyle::Cooked) => (Pat::Str("\""), Pat::Str("\"")), LitKind::Str(_, StrStyle::Raw(0)) => (Pat::Str("r"), Pat::Str("\"")), LitKind::Str(_, StrStyle::Raw(_)) => (Pat::Str("r#"), Pat::Str("#")), - LitKind::ByteStr(_) => (Pat::Str("b\""), Pat::Str("\"")), + LitKind::ByteStr(_, StrStyle::Cooked) => (Pat::Str("b\""), Pat::Str("\"")), + LitKind::ByteStr(_, StrStyle::Raw(0)) => (Pat::Str("br\""), Pat::Str("\"")), + LitKind::ByteStr(_, StrStyle::Raw(_)) => (Pat::Str("br#\""), Pat::Str("#")), LitKind::Byte(_) => (Pat::Str("b'"), Pat::Str("'")), LitKind::Char(_) => (Pat::Str("'"), Pat::Str("'")), LitKind::Int(_, LitIntType::Signed(IntTy::Isize)) => (Pat::Num, Pat::Str("isize")), diff --git a/src/tools/clippy/clippy_utils/src/consts.rs b/src/tools/clippy/clippy_utils/src/consts.rs index 315aea9aa09..7a637d32bab 100644 --- a/src/tools/clippy/clippy_utils/src/consts.rs +++ b/src/tools/clippy/clippy_utils/src/consts.rs @@ -210,7 +210,7 @@ pub fn lit_to_mir_constant(lit: &LitKind, ty: Option<Ty<'_>>) -> Constant { match *lit { LitKind::Str(ref is, _) => Constant::Str(is.to_string()), LitKind::Byte(b) => Constant::Int(u128::from(b)), - LitKind::ByteStr(ref s) => Constant::Binary(Lrc::clone(s)), + LitKind::ByteStr(ref s, _) => Constant::Binary(Lrc::clone(s)), LitKind::Char(c) => Constant::Char(c), LitKind::Int(n, _) => Constant::Int(n), LitKind::Float(ref is, LitFloatType::Suffixed(fty)) => match fty { diff --git a/src/tools/rustfmt/src/attr.rs b/src/tools/rustfmt/src/attr.rs index 2ac703b957b..c503eeeb9b3 100644 --- a/src/tools/rustfmt/src/attr.rs +++ b/src/tools/rustfmt/src/attr.rs @@ -260,7 +260,9 @@ impl Rewrite for ast::NestedMetaItem { fn rewrite(&self, context: &RewriteContext<'_>, shape: Shape) -> Option<String> { match self { ast::NestedMetaItem::MetaItem(ref meta_item) => meta_item.rewrite(context, shape), - ast::NestedMetaItem::Lit(ref l) => rewrite_literal(context, l.token_lit, l.span, shape), + ast::NestedMetaItem::Lit(ref l) => { + rewrite_literal(context, l.as_token_lit(), l.span, shape) + } } } } @@ -308,18 +310,18 @@ impl Rewrite for ast::MetaItem { }), )? } - ast::MetaItemKind::NameValue(ref literal) => { + ast::MetaItemKind::NameValue(ref lit) => { let path = rewrite_path(context, PathContext::Type, &None, &self.path, shape)?; // 3 = ` = ` let lit_shape = shape.shrink_left(path.len() + 3)?; - // `rewrite_literal` returns `None` when `literal` exceeds max + // `rewrite_literal` returns `None` when `lit` exceeds max // width. Since a literal is basically unformattable unless it // is a string literal (and only if `format_strings` is set), // we might be better off ignoring the fact that the attribute // is longer than the max width and continue on formatting. // See #2479 for example. - let value = rewrite_literal(context, literal.token_lit, literal.span, lit_shape) - .unwrap_or_else(|| context.snippet(literal.span).to_owned()); + let value = rewrite_literal(context, lit.as_token_lit(), lit.span, lit_shape) + .unwrap_or_else(|| context.snippet(lit.span).to_owned()); format!("{} = {}", path, value) } }) |