From 478f8287c0e2c35cda511fd3ac01b7ac78ee7cfe Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 2 Jun 2025 08:59:29 +1000 Subject: Introduce `ByteSymbol`. It's like `Symbol` but for byte strings. The interner is now used for both `Symbol` and `ByteSymbol`. E.g. if you intern `"dog"` and `b"dog"` you'll get a `Symbol` and a `ByteSymbol` with the same index and the characters will only be stored once. The motivation for this is to eliminate the `Arc`s in `ast::LitKind`, to make `ast::LitKind` impl `Copy`, and to avoid the need to arena-allocate `ast::LitKind` in HIR. The latter change reduces peak memory by a non-trivial amount on literal-heavy benchmarks such as `deep-vector` and `tuple-stress`. `Encoder`, `Decoder`, `SpanEncoder`, and `SpanDecoder` all get some changes so that they can handle normal strings and byte strings. This change does slow down compilation of programs that use `include_bytes!` on large files, because the contents of those files are now interned (hashed). This makes `include_bytes!` more similar to `include_str!`, though `include_bytes!` contents still aren't escaped, and hashing is still much cheaper than escaping. --- compiler/rustc_expand/src/proc_macro_server.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'compiler/rustc_expand/src') diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index fb5abaefb57..af91c8b8f00 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -599,8 +599,12 @@ impl server::TokenStream for Rustc<'_, '_> { ast::ExprKind::Lit(token_lit) => { Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span)) } - ast::ExprKind::IncludedBytes(bytes) => { - let lit = token::Lit::new(token::ByteStr, escape_byte_str_symbol(bytes), None); + ast::ExprKind::IncludedBytes(byte_sym) => { + let lit = token::Lit::new( + token::ByteStr, + escape_byte_str_symbol(byte_sym.as_byte_str()), + None, + ); Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span)) } ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { -- cgit 1.4.1-3-g733a5