Auto merge of #103812 - clubby789:improve-include-bytes, r=petrochenkov

Delay `include_bytes` to AST lowering

Hopefully addresses #65818.
This PR introduces a new `ExprKind::IncludedBytes` which stores the path and bytes of a file included with `include_bytes!()`. We can then create a literal from the bytes during AST lowering, which means we don't need to escape the bytes into valid UTF8 which is the cause of most of the overhead of embedding large binary blobs.

4 files changed, 21 insertions, 1 deletions

diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs
index bb683936026..f72cd14bea0 100644
--- a/compiler/rustc_builtin_macros/src/assert/context.rs
+++ b/compiler/rustc_builtin_macros/src/assert/context.rs
@@ -303,6 +303,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
             | ExprKind::Field(_, _)
             | ExprKind::ForLoop(_, _, _, _)
             | ExprKind::If(_, _, _)
+            | ExprKind::IncludedBytes(..)
             | ExprKind::InlineAsm(_)
             | ExprKind::Let(_, _, _)
             | ExprKind::Lit(_)
diff --git a/compiler/rustc_builtin_macros/src/concat.rs b/compiler/rustc_builtin_macros/src/concat.rs
index 41f4e8c234d..01454d0e98e 100644
--- a/compiler/rustc_builtin_macros/src/concat.rs
+++ b/compiler/rustc_builtin_macros/src/concat.rs
@@ -43,6 +43,9 @@ pub fn expand_concat(
                     has_errors = true;
                 }
             },
+            ast::ExprKind::IncludedBytes(..) => {
+                cx.span_err(e.span, "cannot concatenate a byte string literal")
+            }
             ast::ExprKind::Err => {
                 has_errors = true;
             }
diff --git a/compiler/rustc_builtin_macros/src/concat_bytes.rs b/compiler/rustc_builtin_macros/src/concat_bytes.rs
index 66e86bf2182..4886ca786a5 100644
--- a/compiler/rustc_builtin_macros/src/concat_bytes.rs
+++ b/compiler/rustc_builtin_macros/src/concat_bytes.rs
@@ -108,6 +108,16 @@ fn handle_array_element(
                 None
             }
         },
+        ast::ExprKind::IncludedBytes(..) => {
+            if !*has_errors {
+                cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
+                    .note("byte strings are treated as arrays of bytes")
+                    .help("try flattening the array")
+                    .emit();
+            }
+            *has_errors = true;
+            None
+        }
         _ => {
             missing_literals.push(expr.span);
             None
@@ -167,6 +177,9 @@ pub fn expand_concat_bytes(
                     has_errors = true;
                 }
             },
+            ast::ExprKind::IncludedBytes(ref bytes) => {
+                accumulator.extend_from_slice(bytes);
+            }
             ast::ExprKind::Err => {
                 has_errors = true;
             }
diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs
index d78bbc3c932..3411bd40c9d 100644
--- a/compiler/rustc_builtin_macros/src/source_util.rs
+++ b/compiler/rustc_builtin_macros/src/source_util.rs
@@ -216,7 +216,10 @@ pub fn expand_include_bytes(
         }
     };
     match cx.source_map().load_binary_file(&file) {
-        Ok(bytes) => base::MacEager::expr(cx.expr_byte_str(sp, bytes)),
+        Ok(bytes) => {
+            let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(bytes.into()));
+            base::MacEager::expr(expr)
+        }
         Err(e) => {
             cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
             DummyResult::any(sp)