diff options
Diffstat (limited to 'compiler')
94 files changed, 2129 insertions, 703 deletions
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 2c3272dccb4..bddb50568d4 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -3367,7 +3367,7 @@ impl TryFrom<ItemKind> for ForeignItemKind { pub type ForeignItem = Item<ForeignItemKind>; // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 5b41ac8a69e..dcdd44c6041 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -1023,7 +1023,7 @@ where } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index f3249f3e5a8..880f92bbe7b 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -768,7 +768,7 @@ impl DelimSpacing { } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs index d2c3b8b2d56..ca4604c60c5 100644 --- a/compiler/rustc_ast_lowering/src/format.rs +++ b/compiler/rustc_ast_lowering/src/format.rs @@ -20,10 +20,126 @@ impl<'hir> LoweringContext<'_, 'hir> { let mut fmt = Cow::Borrowed(fmt); if self.tcx.sess.opts.unstable_opts.flatten_format_args { fmt = flatten_format_args(fmt); - fmt = inline_literals(fmt); + fmt = self.inline_literals(fmt); } expand_format_args(self, sp, &fmt, allow_const) } + + /// Try to convert a literal into an interned string + fn try_inline_lit(&self, lit: token::Lit) -> Option<Symbol> { + match LitKind::from_token_lit(lit) { + Ok(LitKind::Str(s, _)) => Some(s), + Ok(LitKind::Int(n, ty)) => { + match ty { + // unsuffixed integer literals are assumed to be i32's + LitIntType::Unsuffixed => { + (n <= i32::MAX as u128).then_some(Symbol::intern(&n.to_string())) + } + LitIntType::Signed(int_ty) => { + let max_literal = self.int_ty_max(int_ty); + (n <= max_literal).then_some(Symbol::intern(&n.to_string())) + } + LitIntType::Unsigned(uint_ty) => { + let max_literal = self.uint_ty_max(uint_ty); + (n <= max_literal).then_some(Symbol::intern(&n.to_string())) + } + } + } + _ => None, + } + } + + /// Get the maximum value of int_ty. It is platform-dependent due to the byte size of isize + fn int_ty_max(&self, int_ty: IntTy) -> u128 { + match int_ty { + IntTy::Isize => self.tcx.data_layout.pointer_size.signed_int_max() as u128, + IntTy::I8 => i8::MAX as u128, + IntTy::I16 => i16::MAX as u128, + IntTy::I32 => i32::MAX as u128, + IntTy::I64 => i64::MAX as u128, + IntTy::I128 => i128::MAX as u128, + } + } + + /// Get the maximum value of uint_ty. It is platform-dependent due to the byte size of usize + fn uint_ty_max(&self, uint_ty: UintTy) -> u128 { + match uint_ty { + UintTy::Usize => self.tcx.data_layout.pointer_size.unsigned_int_max(), + UintTy::U8 => u8::MAX as u128, + UintTy::U16 => u16::MAX as u128, + UintTy::U32 => u32::MAX as u128, + UintTy::U64 => u64::MAX as u128, + UintTy::U128 => u128::MAX as u128, + } + } + + /// Inline literals into the format string. + /// + /// Turns + /// + /// `format_args!("Hello, {}! {} {}", "World", 123, x)` + /// + /// into + /// + /// `format_args!("Hello, World! 123 {}", x)`. + fn inline_literals<'fmt>(&self, mut fmt: Cow<'fmt, FormatArgs>) -> Cow<'fmt, FormatArgs> { + let mut was_inlined = vec![false; fmt.arguments.all_args().len()]; + let mut inlined_anything = false; + + for i in 0..fmt.template.len() { + let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i] else { continue }; + let Ok(arg_index) = placeholder.argument.index else { continue }; + + let mut literal = None; + + if let FormatTrait::Display = placeholder.format_trait + && placeholder.format_options == Default::default() + && let arg = fmt.arguments.all_args()[arg_index].expr.peel_parens_and_refs() + && let ExprKind::Lit(lit) = arg.kind + { + literal = self.try_inline_lit(lit); + } + + if let Some(literal) = literal { + // Now we need to mutate the outer FormatArgs. + // If this is the first time, this clones the outer FormatArgs. + let fmt = fmt.to_mut(); + // Replace the placeholder with the literal. + fmt.template[i] = FormatArgsPiece::Literal(literal); + was_inlined[arg_index] = true; + inlined_anything = true; + } + } + + // Remove the arguments that were inlined. + if inlined_anything { + let fmt = fmt.to_mut(); + + let mut remove = was_inlined; + + // Don't remove anything that's still used. + for_all_argument_indexes(&mut fmt.template, |index| remove[*index] = false); + + // Drop all the arguments that are marked for removal. + let mut remove_it = remove.iter(); + fmt.arguments.all_args_mut().retain(|_| remove_it.next() != Some(&true)); + + // Calculate the mapping of old to new indexes for the remaining arguments. + let index_map: Vec<usize> = remove + .into_iter() + .scan(0, |i, remove| { + let mapped = *i; + *i += !remove as usize; + Some(mapped) + }) + .collect(); + + // Correct the indexes that refer to arguments that have shifted position. + for_all_argument_indexes(&mut fmt.template, |index| *index = index_map[*index]); + } + + fmt + } } /// Flattens nested `format_args!()` into one. @@ -103,82 +219,6 @@ fn flatten_format_args(mut fmt: Cow<'_, FormatArgs>) -> Cow<'_, FormatArgs> { fmt } -/// Inline literals into the format string. -/// -/// Turns -/// -/// `format_args!("Hello, {}! {} {}", "World", 123, x)` -/// -/// into -/// -/// `format_args!("Hello, World! 123 {}", x)`. -fn inline_literals(mut fmt: Cow<'_, FormatArgs>) -> Cow<'_, FormatArgs> { - let mut was_inlined = vec![false; fmt.arguments.all_args().len()]; - let mut inlined_anything = false; - - for i in 0..fmt.template.len() { - let FormatArgsPiece::Placeholder(placeholder) = &fmt.template[i] else { continue }; - let Ok(arg_index) = placeholder.argument.index else { continue }; - - let mut literal = None; - - if let FormatTrait::Display = placeholder.format_trait - && placeholder.format_options == Default::default() - && let arg = fmt.arguments.all_args()[arg_index].expr.peel_parens_and_refs() - && let ExprKind::Lit(lit) = arg.kind - { - if let token::LitKind::Str | token::LitKind::StrRaw(_) = lit.kind - && let Ok(LitKind::Str(s, _)) = LitKind::from_token_lit(lit) - { - literal = Some(s); - } else if let token::LitKind::Integer = lit.kind - && let Ok(LitKind::Int(n, _)) = LitKind::from_token_lit(lit) - { - literal = Some(Symbol::intern(&n.to_string())); - } - } - - if let Some(literal) = literal { - // Now we need to mutate the outer FormatArgs. - // If this is the first time, this clones the outer FormatArgs. - let fmt = fmt.to_mut(); - // Replace the placeholder with the literal. - fmt.template[i] = FormatArgsPiece::Literal(literal); - was_inlined[arg_index] = true; - inlined_anything = true; - } - } - - // Remove the arguments that were inlined. - if inlined_anything { - let fmt = fmt.to_mut(); - - let mut remove = was_inlined; - - // Don't remove anything that's still used. - for_all_argument_indexes(&mut fmt.template, |index| remove[*index] = false); - - // Drop all the arguments that are marked for removal. - let mut remove_it = remove.iter(); - fmt.arguments.all_args_mut().retain(|_| remove_it.next() != Some(&true)); - - // Calculate the mapping of old to new indexes for the remaining arguments. - let index_map: Vec<usize> = remove - .into_iter() - .scan(0, |i, remove| { - let mapped = *i; - *i += !remove as usize; - Some(mapped) - }) - .collect(); - - // Correct the indexes that refer to arguments that have shifted position. - for_all_argument_indexes(&mut fmt.template, |index| *index = index_map[*index]); - } - - fmt -} - #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] enum ArgumentType { Format(FormatTrait), diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 293e65ce872..2c176828c84 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -3,11 +3,12 @@ //! Note that HIR pretty printing is layered on top of this crate. mod expr; +mod fixup; mod item; use crate::pp::Breaks::{Consistent, Inconsistent}; use crate::pp::{self, Breaks}; -use crate::pprust::state::expr::FixupContext; +use crate::pprust::state::fixup::FixupContext; use ast::TraitBoundModifiers; use rustc_ast::attr::AttrIdGenerator; use rustc_ast::ptr::P; @@ -15,7 +16,6 @@ use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, To use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree}; use rustc_ast::util::classify; use rustc_ast::util::comments::{Comment, CommentStyle}; -use rustc_ast::util::parser; use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, BlockCheckMode, PatKind}; use rustc_ast::{attr, BindingMode, ByRef, DelimArgs, RangeEnd, RangeSyntax, Term}; use rustc_ast::{GenericArg, GenericBound, SelfKind}; @@ -1252,22 +1252,14 @@ impl<'a> State<'a> { ast::StmtKind::Item(item) => self.print_item(item), ast::StmtKind::Expr(expr) => { self.space_if_not_bol(); - self.print_expr_outer_attr_style( - expr, - false, - FixupContext { stmt: true, ..FixupContext::default() }, - ); + self.print_expr_outer_attr_style(expr, false, FixupContext::new_stmt()); if classify::expr_requires_semi_to_be_stmt(expr) { self.word(";"); } } ast::StmtKind::Semi(expr) => { self.space_if_not_bol(); - self.print_expr_outer_attr_style( - expr, - false, - FixupContext { stmt: true, ..FixupContext::default() }, - ); + self.print_expr_outer_attr_style(expr, false, FixupContext::new_stmt()); self.word(";"); } ast::StmtKind::Empty => { @@ -1319,11 +1311,7 @@ impl<'a> State<'a> { ast::StmtKind::Expr(expr) if i == blk.stmts.len() - 1 => { self.maybe_print_comment(st.span.lo()); self.space_if_not_bol(); - self.print_expr_outer_attr_style( - expr, - false, - FixupContext { stmt: true, ..FixupContext::default() }, - ); + self.print_expr_outer_attr_style(expr, false, FixupContext::new_stmt()); self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi())); } _ => self.print_stmt(st), @@ -1367,8 +1355,7 @@ impl<'a> State<'a> { self.word_space("="); self.print_expr_cond_paren( expr, - fixup.parenthesize_exterior_struct_lit && parser::contains_exterior_struct_lit(expr) - || parser::needs_par_as_let_scrutinee(expr.precedence().order()), + fixup.needs_par_as_let_scrutinee(expr), FixupContext::default(), ); } diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs index 6eff70410cb..4cbdc9f256d 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs @@ -1,10 +1,10 @@ use crate::pp::Breaks::Inconsistent; +use crate::pprust::state::fixup::FixupContext; use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT}; use ast::{ForLoopKind, MatchKind}; use itertools::{Itertools, Position}; use rustc_ast::ptr::P; use rustc_ast::token; -use rustc_ast::util::classify; use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast::util::parser::{self, AssocOp, Fixity}; use rustc_ast::{self as ast, BlockCheckMode}; @@ -14,78 +14,6 @@ use rustc_ast::{ }; use std::fmt::Write; -#[derive(Copy, Clone, Debug)] -pub(crate) struct FixupContext { - /// Print expression such that it can be parsed back as a statement - /// consisting of the original expression. - /// - /// The effect of this is for binary operators in statement position to set - /// `leftmost_subexpression_in_stmt` when printing their left-hand operand. - /// - /// ```ignore (illustrative) - /// (match x {}) - 1; // match needs parens when LHS of binary operator - /// - /// match x {}; // not when its own statement - /// ``` - pub stmt: bool, - - /// This is the difference between: - /// - /// ```ignore (illustrative) - /// (match x {}) - 1; // subexpression needs parens - /// - /// let _ = match x {} - 1; // no parens - /// ``` - /// - /// There are 3 distinguishable contexts in which `print_expr` might be - /// called with the expression `$match` as its argument, where `$match` - /// represents an expression of kind `ExprKind::Match`: - /// - /// - stmt=false leftmost_subexpression_in_stmt=false - /// - /// Example: `let _ = $match - 1;` - /// - /// No parentheses required. - /// - /// - stmt=false leftmost_subexpression_in_stmt=true - /// - /// Example: `$match - 1;` - /// - /// Must parenthesize `($match)`, otherwise parsing back the output as a - /// statement would terminate the statement after the closing brace of - /// the match, parsing `-1;` as a separate statement. - /// - /// - stmt=true leftmost_subexpression_in_stmt=false - /// - /// Example: `$match;` - /// - /// No parentheses required. - pub leftmost_subexpression_in_stmt: bool, - - /// This is the difference between: - /// - /// ```ignore (illustrative) - /// if let _ = (Struct {}) {} // needs parens - /// - /// match () { - /// () if let _ = Struct {} => {} // no parens - /// } - /// ``` - pub parenthesize_exterior_struct_lit: bool, -} - -/// The default amount of fixing is minimal fixing. Fixups should be turned on -/// in a targeted fashion where needed. -impl Default for FixupContext { - fn default() -> Self { - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - parenthesize_exterior_struct_lit: false, - } - } -} - impl<'a> State<'a> { fn print_else(&mut self, els: Option<&ast::Expr>) { if let Some(_else) = els { @@ -136,9 +64,7 @@ impl<'a> State<'a> { /// Prints an expr using syntax that's acceptable in a condition position, such as the `cond` in /// `if cond { ... }`. fn print_expr_as_cond(&mut self, expr: &ast::Expr) { - let fixup = - FixupContext { parenthesize_exterior_struct_lit: true, ..FixupContext::default() }; - self.print_expr_cond_paren(expr, Self::cond_needs_par(expr), fixup) + self.print_expr_cond_paren(expr, Self::cond_needs_par(expr), FixupContext::new_cond()) } /// Does `expr` need parentheses when printed in a condition position? @@ -310,15 +236,7 @@ impl<'a> State<'a> { // because the latter is valid syntax but with the incorrect meaning. // It's a match-expression followed by tuple-expression, not a function // call. - self.print_expr_maybe_paren( - func, - prec, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(func, prec, fixup.leftmost_subexpression()); self.print_call_post(args) } @@ -387,33 +305,17 @@ impl<'a> State<'a> { _ => left_prec, }; - self.print_expr_maybe_paren( - lhs, - left_prec, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(lhs, left_prec, fixup.leftmost_subexpression()); self.space(); self.word_space(op.node.as_str()); - self.print_expr_maybe_paren( - rhs, - right_prec, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, - ); + self.print_expr_maybe_paren(rhs, right_prec, fixup.subsequent_subexpression()); } fn print_expr_unary(&mut self, op: ast::UnOp, expr: &ast::Expr, fixup: FixupContext) { self.word(op.as_str()); - self.print_expr_maybe_paren( - expr, - parser::PREC_PREFIX, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, - ); + self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup.subsequent_subexpression()); } fn print_expr_addr_of( @@ -431,11 +333,7 @@ impl<'a> State<'a> { self.print_mutability(mutability, true); } } - self.print_expr_maybe_paren( - expr, - parser::PREC_PREFIX, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, - ); + self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup.subsequent_subexpression()); } pub(super) fn print_expr(&mut self, expr: &ast::Expr, fixup: FixupContext) { @@ -470,8 +368,7 @@ impl<'a> State<'a> { // // Same applies to a small set of other expression kinds which eagerly // terminate a statement which opens with them. - let needs_par = - fixup.leftmost_subexpression_in_stmt && !classify::expr_requires_semi_to_be_stmt(expr); + let needs_par = fixup.would_cause_statement_boundary(expr); if needs_par { self.popen(); fixup = FixupContext::default(); @@ -519,16 +416,7 @@ impl<'a> State<'a> { } ast::ExprKind::Cast(expr, ty) => { let prec = AssocOp::As.precedence() as i8; - self.print_expr_maybe_paren( - expr, - prec, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt - || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(expr, prec, fixup.leftmost_subexpression()); self.space(); self.word_space("as"); self.print_type(ty); @@ -660,70 +548,34 @@ impl<'a> State<'a> { self.print_block_with_attrs(blk, attrs); } ast::ExprKind::Await(expr, _) => { - // Same fixups as ExprKind::MethodCall. self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup); self.word(".await"); } ast::ExprKind::Assign(lhs, rhs, _) => { - // Same fixups as ExprKind::Binary. let prec = AssocOp::Assign.precedence() as i8; - self.print_expr_maybe_paren( - lhs, - prec + 1, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt - || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(lhs, prec + 1, fixup.leftmost_subexpression()); self.space(); self.word_space("="); - self.print_expr_maybe_paren( - rhs, - prec, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, - ); + self.print_expr_maybe_paren(rhs, prec, fixup.subsequent_subexpression()); } ast::ExprKind::AssignOp(op, lhs, rhs) => { - // Same fixups as ExprKind::Binary. let prec = AssocOp::Assign.precedence() as i8; - self.print_expr_maybe_paren( - lhs, - prec + 1, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt - || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(lhs, prec + 1, fixup.leftmost_subexpression()); self.space(); self.word(op.node.as_str()); self.word_space("="); - self.print_expr_maybe_paren( - rhs, - prec, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, - ); + self.print_expr_maybe_paren(rhs, prec, fixup.subsequent_subexpression()); } ast::ExprKind::Field(expr, ident) => { - // Same fixups as ExprKind::MethodCall. self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup); self.word("."); self.print_ident(*ident); } ast::ExprKind::Index(expr, index, _) => { - // Same fixups as ExprKind::Call. self.print_expr_maybe_paren( expr, parser::PREC_POSTFIX, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt - || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, + fixup.leftmost_subexpression(), ); self.word("["); self.print_expr(index, FixupContext::default()); @@ -736,31 +588,14 @@ impl<'a> State<'a> { // a "normal" binop gets parenthesized. (`LOr` is the lowest-precedence binop.) let fake_prec = AssocOp::LOr.precedence() as i8; if let Some(e) = start { - self.print_expr_maybe_paren( - e, - fake_prec, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: fixup.stmt - || fixup.leftmost_subexpression_in_stmt, - ..fixup - }, - ); + self.print_expr_maybe_paren(e, fake_prec, fixup.leftmost_subexpression()); } match limits { ast::RangeLimits::HalfOpen => self.word(".."), ast::RangeLimits::Closed => self.word("..="), } if let Some(e) = end { - self.print_expr_maybe_paren( - e, - fake_prec, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - ..fixup - }, - ); + self.print_expr_maybe_paren(e, fake_prec, fixup.subsequent_subexpression()); } } ast::ExprKind::Underscore => self.word("_"), @@ -777,11 +612,7 @@ impl<'a> State<'a> { self.print_expr_maybe_paren( expr, parser::PREC_JUMP, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - ..fixup - }, + fixup.subsequent_subexpression(), ); } } @@ -799,11 +630,7 @@ impl<'a> State<'a> { self.print_expr_maybe_paren( expr, parser::PREC_JUMP, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - ..fixup - }, + fixup.subsequent_subexpression(), ); } } @@ -816,11 +643,7 @@ impl<'a> State<'a> { self.print_expr_maybe_paren( expr, parser::PREC_JUMP, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - ..fixup - }, + fixup.subsequent_subexpression(), ); } } @@ -830,7 +653,7 @@ impl<'a> State<'a> { self.print_expr_maybe_paren( result, parser::PREC_JUMP, - FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..fixup }, + fixup.subsequent_subexpression(), ); } ast::ExprKind::InlineAsm(a) => { @@ -884,16 +707,11 @@ impl<'a> State<'a> { self.print_expr_maybe_paren( expr, parser::PREC_JUMP, - FixupContext { - stmt: false, - leftmost_subexpression_in_stmt: false, - ..fixup - }, + fixup.subsequent_subexpression(), ); } } ast::ExprKind::Try(e) => { - // Same fixups as ExprKind::MethodCall. self.print_expr_maybe_paren(e, parser::PREC_POSTFIX, fixup); self.word("?") } @@ -961,7 +779,7 @@ impl<'a> State<'a> { } _ => { self.end(); // Close the ibox for the pattern. - self.print_expr(body, FixupContext { stmt: true, ..FixupContext::default() }); + self.print_expr(body, FixupContext::new_stmt()); self.word(","); } } diff --git a/compiler/rustc_ast_pretty/src/pprust/state/fixup.rs b/compiler/rustc_ast_pretty/src/pprust/state/fixup.rs new file mode 100644 index 00000000000..d21cb82f83b --- /dev/null +++ b/compiler/rustc_ast_pretty/src/pprust/state/fixup.rs @@ -0,0 +1,149 @@ +use rustc_ast::util::{classify, parser}; +use rustc_ast::Expr; + +#[derive(Copy, Clone, Debug)] +pub(crate) struct FixupContext { + /// Print expression such that it can be parsed back as a statement + /// consisting of the original expression. + /// + /// The effect of this is for binary operators in statement position to set + /// `leftmost_subexpression_in_stmt` when printing their left-hand operand. + /// + /// ```ignore (illustrative) + /// (match x {}) - 1; // match needs parens when LHS of binary operator + /// + /// match x {}; // not when its own statement + /// ``` + stmt: bool, + + /// This is the difference between: + /// + /// ```ignore (illustrative) + /// (match x {}) - 1; // subexpression needs parens + /// + /// let _ = match x {} - 1; // no parens + /// ``` + /// + /// There are 3 distinguishable contexts in which `print_expr` might be + /// called with the expression `$match` as its argument, where `$match` + /// represents an expression of kind `ExprKind::Match`: + /// + /// - stmt=false leftmost_subexpression_in_stmt=false + /// + /// Example: `let _ = $match - 1;` + /// + /// No parentheses required. + /// + /// - stmt=false leftmost_subexpression_in_stmt=true + /// + /// Example: `$match - 1;` + /// + /// Must parenthesize `($match)`, otherwise parsing back the output as a + /// statement would terminate the statement after the closing brace of + /// the match, parsing `-1;` as a separate statement. + /// + /// - stmt=true leftmost_subexpression_in_stmt=false + /// + /// Example: `$match;` + /// + /// No parentheses required. + leftmost_subexpression_in_stmt: bool, + + /// This is the difference between: + /// + /// ```ignore (illustrative) + /// if let _ = (Struct {}) {} // needs parens + /// + /// match () { + /// () if let _ = Struct {} => {} // no parens + /// } + /// ``` + parenthesize_exterior_struct_lit: bool, +} + +/// The default amount of fixing is minimal fixing. Fixups should be turned on +/// in a targeted fashion where needed. +impl Default for FixupContext { + fn default() -> Self { + FixupContext { + stmt: false, + leftmost_subexpression_in_stmt: false, + parenthesize_exterior_struct_lit: false, + } + } +} + +impl FixupContext { + /// Create the initial fixup for printing an expression in statement + /// position. + /// + /// This is currently also used for printing an expression as a match-arm, + /// but this is incorrect and leads to over-parenthesizing. + pub fn new_stmt() -> Self { + FixupContext { stmt: true, ..FixupContext::default() } + } + + /// Create the initial fixup for printing an expression as the "condition" + /// of an `if` or `while`. There are a few other positions which are + /// grammatically equivalent and also use this, such as the iterator + /// expression in `for` and the scrutinee in `match`. + pub fn new_cond() -> Self { + FixupContext { parenthesize_exterior_struct_lit: true, ..FixupContext::default() } + } + + /// Transform this fixup into the one that should apply when printing the + /// leftmost subexpression of the current expression. + /// + /// The leftmost subexpression is any subexpression that has the same first + /// token as the current expression, but has a different last token. + /// + /// For example in `$a + $b` and `$a.method()`, the subexpression `$a` is a + /// leftmost subexpression. + /// + /// Not every expression has a leftmost subexpression. For example neither + /// `-$a` nor `[$a]` have one. + pub fn leftmost_subexpression(self) -> Self { + FixupContext { + stmt: false, + leftmost_subexpression_in_stmt: self.stmt || self.leftmost_subexpression_in_stmt, + ..self + } + } + + /// Transform this fixup into the one that should apply when printing any + /// subexpression that is neither a leftmost subexpression nor surrounded in + /// delimiters. + /// + /// This is for any subexpression that has a different first token than the + /// current expression, and is not surrounded by a paren/bracket/brace. For + /// example the `$b` in `$a + $b` and `-$b`, but not the one in `[$b]` or + /// `$a.f($b)`. + pub fn subsequent_subexpression(self) -> Self { + FixupContext { stmt: false, leftmost_subexpression_in_stmt: false, ..self } + } + + /// Determine whether parentheses are needed around the given expression to + /// head off an unintended statement boundary. + /// + /// The documentation on `FixupContext::leftmost_subexpression_in_stmt` has + /// examples. + pub fn would_cause_statement_boundary(self, expr: &Expr) -> bool { + self.leftmost_subexpression_in_stmt && !classify::expr_requires_semi_to_be_stmt(expr) + } + + /// Determine whether parentheses are needed around the given `let` + /// scrutinee. + /// + /// In `if let _ = $e {}`, some examples of `$e` that would need parentheses + /// are: + /// + /// - `Struct {}.f()`, because otherwise the `{` would be misinterpreted + /// as the opening of the if's then-block. + /// + /// - `true && false`, because otherwise this would be misinterpreted as a + /// "let chain". + pub fn needs_par_as_let_scrutinee(self, expr: &Expr) -> bool { + self.parenthesize_exterior_struct_lit && parser::contains_exterior_struct_lit(expr) + || parser::needs_par_as_let_scrutinee(expr.precedence().order()) + } +} diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs index 13f27c1c95c..10886aace53 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs @@ -1,5 +1,5 @@ use crate::pp::Breaks::Inconsistent; -use crate::pprust::state::expr::FixupContext; +use crate::pprust::state::fixup::FixupContext; use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT}; use ast::StaticItem; diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs index 8b0b9123ac7..efa4be7e15a 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs @@ -1,4 +1,13 @@ -#![feature(no_core, lang_items, never_type, linkage, extern_types, thread_local, repr_simd)] +#![feature( + no_core, + lang_items, + never_type, + linkage, + extern_types, + thread_local, + repr_simd, + raw_ref_op +)] #![no_core] #![allow(dead_code, non_camel_case_types, internal_features)] @@ -112,9 +121,7 @@ fn start<T: Termination + 'static>( static mut NUM: u8 = 6 * 7; -// FIXME: Use `SyncUnsafeCell` instead of allowing `static_mut_refs` lint -#[allow(static_mut_refs)] -static NUM_REF: &'static u8 = unsafe { &NUM }; +static NUM_REF: &'static u8 = unsafe { &*&raw const NUM }; unsafe fn zeroed<T>() -> T { let mut uninit = MaybeUninit { uninit: () }; diff --git a/compiler/rustc_codegen_cranelift/patches/0030-stdlib-Revert-use-raw-dylib-for-Windows-futex-APIs.patch b/compiler/rustc_codegen_cranelift/patches/0030-stdlib-Revert-use-raw-dylib-for-Windows-futex-APIs.patch new file mode 100644 index 00000000000..21f5ee9cc6e --- /dev/null +++ b/compiler/rustc_codegen_cranelift/patches/0030-stdlib-Revert-use-raw-dylib-for-Windows-futex-APIs.patch @@ -0,0 +1,37 @@ +From 0d741cf82c3c908616abd39dc84ebf7d8702e0c3 Mon Sep 17 00:00:00 2001 +From: Chris Denton <chris@chrisdenton.dev> +Date: Tue, 16 Apr 2024 15:51:34 +0000 +Subject: [PATCH] Revert use raw-dylib for Windows futex APIs + +--- + library/std/src/sys/pal/windows/c.rs | 14 +------------- + 1 file changed, 1 insertion(+), 13 deletions(-) + +diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs +index 9d58ce05f01..1c828bac4b6 100644 +--- a/library/std/src/sys/pal/windows/c.rs ++++ b/library/std/src/sys/pal/windows/c.rs +@@ -357,19 +357,7 @@ pub fn GetTempPath2W(bufferlength: u32, buffer: PWSTR) -> u32 { + } + + #[cfg(not(target_vendor = "win7"))] +-// Use raw-dylib to import synchronization functions to workaround issues with the older mingw import library. +-#[cfg_attr( +- target_arch = "x86", +- link( +- name = "api-ms-win-core-synch-l1-2-0", +- kind = "raw-dylib", +- import_name_type = "undecorated" +- ) +-)] +-#[cfg_attr( +- not(target_arch = "x86"), +- link(name = "api-ms-win-core-synch-l1-2-0", kind = "raw-dylib") +-)] ++#[link(name = "synchronization")] + extern "system" { + pub fn WaitOnAddress( + address: *const c_void, +-- +2.42.0.windows.2 + diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 635ed6c8e88..cb05c17ec2a 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -110,7 +110,7 @@ pub(crate) fn codegen_const_value<'tcx>( if fx.clif_type(layout.ty).is_some() { return CValue::const_val(fx, layout, int); } else { - let raw_val = int.size().truncate(int.to_bits(int.size()).unwrap()); + let raw_val = int.size().truncate(int.assert_bits(int.size())); let val = match int.size().bytes() { 1 => fx.bcx.ins().iconst(types::I8, raw_val as i64), 2 => fx.bcx.ins().iconst(types::I16, raw_val as i64), @@ -491,27 +491,24 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( return None; } let scalar_int = mir_operand_get_const_val(fx, operand)?; - let scalar_int = match fx - .layout_of(*ty) - .size - .cmp(&scalar_int.size()) - { - Ordering::Equal => scalar_int, - Ordering::Less => match ty.kind() { - ty::Uint(_) => ScalarInt::try_from_uint( - scalar_int.try_to_uint(scalar_int.size()).unwrap(), - fx.layout_of(*ty).size, - ) - .unwrap(), - ty::Int(_) => ScalarInt::try_from_int( - scalar_int.try_to_int(scalar_int.size()).unwrap(), - fx.layout_of(*ty).size, - ) - .unwrap(), - _ => unreachable!(), - }, - Ordering::Greater => return None, - }; + let scalar_int = + match fx.layout_of(*ty).size.cmp(&scalar_int.size()) { + Ordering::Equal => scalar_int, + Ordering::Less => match ty.kind() { + ty::Uint(_) => ScalarInt::try_from_uint( + scalar_int.assert_uint(scalar_int.size()), + fx.layout_of(*ty).size, + ) + .unwrap(), + ty::Int(_) => ScalarInt::try_from_int( + scalar_int.assert_int(scalar_int.size()), + fx.layout_of(*ty).size, + ) + .unwrap(), + _ => unreachable!(), + }, + Ordering::Greater => return None, + }; computed_scalar_int = Some(scalar_int); } Rvalue::Use(operand) => { diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index fc5b88a54fe..ad863903cee 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -326,7 +326,7 @@ impl<'tcx> CValue<'tcx> { let val = match layout.ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { - let const_val = const_val.to_bits(layout.size).unwrap(); + let const_val = const_val.assert_bits(layout.size); let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64); let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64); fx.bcx.ins().iconcat(lsb, msb) @@ -338,7 +338,7 @@ impl<'tcx> CValue<'tcx> { | ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) => { - let raw_val = const_val.size().truncate(const_val.to_bits(layout.size).unwrap()); + let raw_val = const_val.size().truncate(const_val.assert_bits(layout.size)); fx.bcx.ins().iconst(clif_ty, raw_val as i64) } ty::Float(FloatTy::F32) => { diff --git a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs index add77880716..5a7ddc4cd7f 100644 --- a/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs +++ b/compiler/rustc_codegen_gcc/example/mini_core_hello_world.rs @@ -2,7 +2,7 @@ #![feature( no_core, unboxed_closures, start, lang_items, never_type, linkage, - extern_types, thread_local + extern_types, thread_local, raw_ref_op )] #![no_core] #![allow(dead_code, internal_features, non_camel_case_types)] @@ -99,9 +99,7 @@ fn start<T: Termination + 'static>( static mut NUM: u8 = 6 * 7; -// FIXME: Use `SyncUnsafeCell` instead of allowing `static_mut_refs` lint -#[allow(static_mut_refs)] -static NUM_REF: &'static u8 = unsafe { &NUM }; +static NUM_REF: &'static u8 = unsafe { &* &raw const NUM }; macro_rules! assert { ($e:expr) => { diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 6253816d37d..d353704fb75 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -31,7 +31,7 @@ use rustc_span::Span; use rustc_target::abi::{ self, call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout, WrappingRange, }; -use rustc_target::spec::{HasTargetSpec, Target}; +use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, WasmCAbi}; use crate::common::{type_is_pointer, SignType, TypeReflection}; use crate::context::CodegenCx; @@ -2352,6 +2352,12 @@ impl<'tcx> HasTargetSpec for Builder<'_, '_, 'tcx> { } } +impl<'tcx> HasWasmCAbiOpt for Builder<'_, '_, 'tcx> { + fn wasm_c_abi_opt(&self) -> WasmCAbi { + self.cx.wasm_c_abi_opt() + } +} + pub trait ToGccComp { fn to_gcc_comparison(&self) -> ComparisonOp; } diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 9e6cf3e34df..16a85b4e8fa 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -20,7 +20,7 @@ use rustc_span::{source_map::respan, Span}; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; -use rustc_target::spec::{HasTargetSpec, Target, TlsModel}; +use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCAbi}; use crate::callee::get_fn; use crate::common::SignType; @@ -557,6 +557,12 @@ impl<'gcc, 'tcx> HasTargetSpec for CodegenCx<'gcc, 'tcx> { } } +impl<'gcc, 'tcx> HasWasmCAbiOpt for CodegenCx<'gcc, 'tcx> { + fn wasm_c_abi_opt(&self) -> WasmCAbi { + self.tcx.sess.opts.unstable_opts.wasm_c_abi + } +} + impl<'gcc, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> { type LayoutOfResult = TyAndLayout<'tcx>; diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 0619000364b..d4a3e39cef7 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -415,6 +415,7 @@ impl<'a> LlvmArchiveBuilder<'a> { members.as_ptr() as *const &_, true, kind, + self.sess.target.arch == "arm64ec", ); let ret = if r.into_result().is_err() { let err = llvm::LLVMRustGetLastError(); diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 160f361b9b5..4c2bdb2f5ec 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -17,7 +17,7 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; @@ -1702,4 +1702,128 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { }; kcfi_bundle } + + pub(crate) fn mcdc_parameters( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + bitmap_bytes: &'ll Value, + ) -> &'ll Value { + debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); + + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + + let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], + self.cx.type_void(), + ); + let args = &[fn_name, hash, bitmap_bytes]; + let args = self.check_call("call", llty, llfn, args); + + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + // Create condition bitmap named `mcdc.addr`. + let mut bx = Builder::with_cx(self.cx); + bx.position_at_start(llvm::LLVMGetFirstBasicBlock(self.llfn())); + let cond_bitmap = { + let alloca = + llvm::LLVMBuildAlloca(bx.llbuilder, bx.cx.type_i32(), c"mcdc.addr".as_ptr()); + llvm::LLVMSetAlignment(alloca, 4); + alloca + }; + bx.store(self.const_i32(0), cond_bitmap, self.tcx().data_layout.i32_align.abi); + cond_bitmap + } + } + + pub(crate) fn mcdc_tvbitmap_update( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + bitmap_bytes: &'ll Value, + bitmap_index: &'ll Value, + mcdc_temp: &'ll Value, + ) { + debug!( + "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", + fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp + ); + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + + let llfn = + unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[ + self.cx.type_ptr(), + self.cx.type_i64(), + self.cx.type_i32(), + self.cx.type_i32(), + self.cx.type_ptr(), + ], + self.cx.type_void(), + ); + let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp]; + let args = self.check_call("call", llty, llfn, args); + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + } + let i32_align = self.tcx().data_layout.i32_align.abi; + self.store(self.const_i32(0), mcdc_temp, i32_align); + } + + pub(crate) fn mcdc_condbitmap_update( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + cond_loc: &'ll Value, + mcdc_temp: &'ll Value, + bool_value: &'ll Value, + ) { + debug!( + "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", + fn_name, hash, cond_loc, mcdc_temp, bool_value + ); + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[ + self.cx.type_ptr(), + self.cx.type_i64(), + self.cx.type_i32(), + self.cx.type_ptr(), + self.cx.type_i1(), + ], + self.cx.type_void(), + ); + let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value]; + self.check_call("call", llty, llfn, args); + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + } + } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 2af28146a51..12a846a49ec 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,6 @@ -use rustc_middle::mir::coverage::{CodeRegion, CounterId, CovTerm, ExpressionId, MappingKind}; +use rustc_middle::mir::coverage::{ + CodeRegion, ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, +}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -99,6 +101,86 @@ pub enum RegionKind { /// associated with two counters, each representing the number of times the /// expression evaluates to true or false. BranchRegion = 4, + + /// A DecisionRegion represents a top-level boolean expression and is + /// associated with a variable length bitmap index and condition number. + MCDCDecisionRegion = 5, + + /// A Branch Region can be extended to include IDs to facilitate MC/DC. + MCDCBranchRegion = 6, +} + +pub mod mcdc { + use rustc_middle::mir::coverage::{ConditionInfo, DecisionInfo}; + + /// Must match the layout of `LLVMRustMCDCDecisionParameters`. + #[repr(C)] + #[derive(Clone, Copy, Debug, Default)] + pub struct DecisionParameters { + bitmap_idx: u32, + conditions_num: u16, + } + + // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at [19](https://github.com/llvm/llvm-project/pull/81257) + type LLVMConditionId = i16; + + /// Must match the layout of `LLVMRustMCDCBranchParameters`. + #[repr(C)] + #[derive(Clone, Copy, Debug, Default)] + pub struct BranchParameters { + condition_id: LLVMConditionId, + condition_ids: [LLVMConditionId; 2], + } + + #[repr(C)] + #[derive(Clone, Copy, Debug)] + pub enum ParameterTag { + None = 0, + Decision = 1, + Branch = 2, + } + /// Same layout with `LLVMRustMCDCParameters` + #[repr(C)] + #[derive(Clone, Copy, Debug)] + pub struct Parameters { + tag: ParameterTag, + decision_params: DecisionParameters, + branch_params: BranchParameters, + } + + impl Parameters { + pub fn none() -> Self { + Self { + tag: ParameterTag::None, + decision_params: Default::default(), + branch_params: Default::default(), + } + } + pub fn decision(decision_params: DecisionParameters) -> Self { + Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } + } + pub fn branch(branch_params: BranchParameters) -> Self { + Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } + } + } + + impl From<ConditionInfo> for BranchParameters { + fn from(value: ConditionInfo) -> Self { + Self { + condition_id: value.condition_id.as_u32() as LLVMConditionId, + condition_ids: [ + value.false_next_id.as_u32() as LLVMConditionId, + value.true_next_id.as_u32() as LLVMConditionId, + ], + } + } + } + + impl From<DecisionInfo> for DecisionParameters { + fn from(value: DecisionInfo) -> Self { + Self { bitmap_idx: value.bitmap_idx, conditions_num: value.conditions_num } + } + } } /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the @@ -122,6 +204,7 @@ pub struct CounterMappingRegion { /// for the false branch of the region. false_counter: Counter, + mcdc_params: mcdc::Parameters, /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes /// that, in turn, are used to look up the filename for this region. @@ -173,6 +256,26 @@ impl CounterMappingRegion { end_line, end_col, ), + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + Self::mcdc_branch_region( + Counter::from_term(true_term), + Counter::from_term(false_term), + mcdc_params, + local_file_id, + start_line, + start_col, + end_line, + end_col, + ) + } + MappingKind::MCDCDecision(decision_info) => Self::decision_region( + decision_info, + local_file_id, + start_line, + start_col, + end_line, + end_col, + ), } } @@ -187,6 +290,7 @@ impl CounterMappingRegion { Self { counter, false_counter: Counter::ZERO, + mcdc_params: mcdc::Parameters::none(), file_id, expanded_file_id: 0, start_line, @@ -209,6 +313,7 @@ impl CounterMappingRegion { Self { counter, false_counter, + mcdc_params: mcdc::Parameters::none(), file_id, expanded_file_id: 0, start_line, @@ -219,6 +324,54 @@ impl CounterMappingRegion { } } + pub(crate) fn mcdc_branch_region( + counter: Counter, + false_counter: Counter, + condition_info: ConditionInfo, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter, + mcdc_params: mcdc::Parameters::branch(condition_info.into()), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::MCDCBranchRegion, + } + } + + pub(crate) fn decision_region( + decision_info: DecisionInfo, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + let mcdc_params = mcdc::Parameters::decision(decision_info.into()); + + Self { + counter: Counter::ZERO, + false_counter: Counter::ZERO, + mcdc_params, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::MCDCDecisionRegion, + } + } + // This function might be used in the future; the LLVM API is still evolving, as is coverage // support. #[allow(dead_code)] @@ -233,6 +386,7 @@ impl CounterMappingRegion { Self { counter: Counter::ZERO, false_counter: Counter::ZERO, + mcdc_params: mcdc::Parameters::none(), file_id, expanded_file_id, start_line, @@ -256,6 +410,7 @@ impl CounterMappingRegion { Self { counter: Counter::ZERO, false_counter: Counter::ZERO, + mcdc_params: mcdc::Parameters::none(), file_id, expanded_file_id: 0, start_line, @@ -280,6 +435,7 @@ impl CounterMappingRegion { Self { counter, false_counter: Counter::ZERO, + mcdc_params: mcdc::Parameters::none(), file_id, expanded_file_id: 0, start_line, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 140566e8da9..085ce15d81f 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::{ use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::CoverageKind; +use rustc_middle::mir::coverage::{CoverageKind, FunctionCoverageInfo}; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; use rustc_target::abi::Align; @@ -30,6 +30,7 @@ pub struct CrateCoverageContext<'ll, 'tcx> { pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, + pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -37,6 +38,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { Self { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), + mcdc_condition_bitmap_map: Default::default(), } } @@ -45,6 +47,12 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { self.function_coverage_map.replace(FxIndexMap::default()) } + + /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is called condition bitmap. + /// This value is named `mcdc.addr` (same as clang) and is a 32-bit integer. + fn try_get_mcdc_condition_bitmap(&self, instance: &Instance<'tcx>) -> Option<&'ll llvm::Value> { + self.mcdc_condition_bitmap_map.borrow().get(instance).copied() + } } // These methods used to be part of trait `CoverageInfoMethods`, which no longer @@ -90,6 +98,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; + if function_coverage_info.mcdc_bitmap_bytes > 0 { + ensure_mcdc_parameters(bx, instance, function_coverage_info); + } + let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map @@ -131,10 +143,66 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } + CoverageKind::CondBitmapUpdate { id, value, .. } => { + drop(coverage_map); + assert_ne!( + id.as_u32(), + 0, + "ConditionId of evaluated conditions should never be zero" + ); + let cond_bitmap = coverage_context + .try_get_mcdc_condition_bitmap(&instance) + .expect("mcdc cond bitmap should have been allocated for updating"); + let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); + let bool_value = bx.const_bool(value); + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); + } + CoverageKind::TestVectorBitmapUpdate { bitmap_idx } => { + drop(coverage_map); + let cond_bitmap = coverage_context + .try_get_mcdc_condition_bitmap(&instance) + .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); + let bitmap_bytes = bx.tcx().coverage_ids_info(instance.def).mcdc_bitmap_bytes; + assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); + assert!( + bitmap_bytes <= function_coverage_info.mcdc_bitmap_bytes, + "bitmap length disagreement: query says {bitmap_bytes} but function info only has {}", + function_coverage_info.mcdc_bitmap_bytes + ); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = bx.const_u32(bitmap_bytes); + let bitmap_index = bx.const_u32(bitmap_idx); + bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap); + } } } } +fn ensure_mcdc_parameters<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + instance: Instance<'tcx>, + function_coverage_info: &FunctionCoverageInfo, +) { + let Some(cx) = bx.coverage_context() else { return }; + if cx.mcdc_condition_bitmap_map.borrow().contains_key(&instance) { + return; + } + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = bx.const_u32(function_coverage_info.mcdc_bitmap_bytes); + let cond_bitmap = bx.mcdc_parameters(fn_name, hash, bitmap_bytes); + bx.coverage_context() + .expect("already checked above") + .mcdc_condition_bitmap_map + .borrow_mut() + .insert(instance, cond_bitmap); +} + /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 5509baaa3e9..a10dc61967e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1631,6 +1631,10 @@ extern "C" { // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustBuildCall<'a>( B: &Builder<'a>, Ty: &'a Type, @@ -2303,6 +2307,7 @@ extern "C" { Members: *const &RustArchiveMember<'_>, WriteSymbtab: bool, Kind: ArchiveKind, + isEC: bool, ) -> LLVMRustResult; pub fn LLVMRustArchiveMemberNew<'a>( Filename: *const c_char, diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs index 704f597cfdb..caacc6f57d3 100644 --- a/compiler/rustc_const_eval/src/interpret/discriminant.rs +++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs @@ -295,8 +295,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { &niche_start_val, )? .to_scalar() - .try_to_int() - .unwrap(); + .assert_int(); Ok(Some((tag, tag_field))) } } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index c120154ce2a..718c91b2f76 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -6,9 +6,10 @@ use std::assert_matches::assert_matches; use either::{Either, Left, Right}; use rustc_hir::def::Namespace; +use rustc_middle::mir::interpret::ScalarSizeMismatch; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter}; -use rustc_middle::ty::{ConstInt, Ty, TyCtxt}; +use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt}; use rustc_middle::{mir, ty}; use rustc_target::abi::{self, Abi, HasDataLayout, Size}; @@ -211,6 +212,12 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { } #[inline] + pub fn from_scalar_int(s: ScalarInt, layout: TyAndLayout<'tcx>) -> Self { + assert_eq!(s.size(), layout.size); + Self::from_scalar(Scalar::from(s), layout) + } + + #[inline] pub fn try_from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Option<Self> { Some(Self::from_scalar(Scalar::try_from_uint(i, layout.size)?, layout)) } @@ -223,7 +230,6 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { pub fn try_from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Option<Self> { Some(Self::from_scalar(Scalar::try_from_int(i, layout.size)?, layout)) } - #[inline] pub fn from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Self { Self::from_scalar(Scalar::from_int(i, layout.size), layout) @@ -242,6 +248,20 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { Self::from_scalar(Scalar::from_i8(c as i8), layout) } + /// Return the immediate as a `ScalarInt`. Ensures that it has the size that the layout of the + /// immediate indicates. + #[inline] + pub fn to_scalar_int(&self) -> InterpResult<'tcx, ScalarInt> { + let s = self.to_scalar().to_scalar_int()?; + if s.size() != self.layout.size { + throw_ub!(ScalarSizeMismatch(ScalarSizeMismatch { + target_size: self.layout.size.bytes(), + data_size: s.size().bytes(), + })); + } + Ok(s) + } + #[inline] pub fn to_const_int(self) -> ConstInt { assert!(self.layout.ty.is_integral()); @@ -792,7 +812,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index 5665bb4999f..9af755e40de 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -2,7 +2,7 @@ use rustc_apfloat::{Float, FloatConvert}; use rustc_middle::mir; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use rustc_middle::ty::{self, FloatTy, Ty}; +use rustc_middle::ty::{self, FloatTy, ScalarInt, Ty}; use rustc_span::symbol::sym; use rustc_target::abi::Abi; @@ -146,14 +146,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { fn binary_int_op( &self, bin_op: mir::BinOp, - // passing in raw bits - l: u128, - left_layout: TyAndLayout<'tcx>, - r: u128, - right_layout: TyAndLayout<'tcx>, + left: &ImmTy<'tcx, M::Provenance>, + right: &ImmTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, (ImmTy<'tcx, M::Provenance>, bool)> { use rustc_middle::mir::BinOp::*; + // This checks the size, so that we can just assert it below. + let l = left.to_scalar_int()?; + let r = right.to_scalar_int()?; + // Prepare to convert the values to signed or unsigned form. + let l_signed = || l.assert_int(left.layout.size); + let l_unsigned = || l.assert_uint(left.layout.size); + let r_signed = || r.assert_int(right.layout.size); + let r_unsigned = || r.assert_uint(right.layout.size); + let throw_ub_on_overflow = match bin_op { AddUnchecked => Some(sym::unchecked_add), SubUnchecked => Some(sym::unchecked_sub), @@ -165,69 +171,72 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Shift ops can have an RHS with a different numeric type. if matches!(bin_op, Shl | ShlUnchecked | Shr | ShrUnchecked) { - let size = left_layout.size.bits(); + let size = left.layout.size.bits(); // The shift offset is implicitly masked to the type size. (This is the one MIR operator // that does *not* directly map to a single LLVM operation.) Compute how much we // actually shift and whether there was an overflow due to shifting too much. - let (shift_amount, overflow) = if right_layout.abi.is_signed() { - let shift_amount = self.sign_extend(r, right_layout) as i128; + let (shift_amount, overflow) = if right.layout.abi.is_signed() { + let shift_amount = r_signed(); let overflow = shift_amount < 0 || shift_amount >= i128::from(size); + // Deliberately wrapping `as` casts: shift_amount *can* be negative, but the result + // of the `as` will be equal modulo `size` (since it is a power of two). let masked_amount = (shift_amount as u128) % u128::from(size); - debug_assert_eq!(overflow, shift_amount != (masked_amount as i128)); + assert_eq!(overflow, shift_amount != (masked_amount as i128)); (masked_amount, overflow) } else { - let shift_amount = r; + let shift_amount = r_unsigned(); let masked_amount = shift_amount % u128::from(size); (masked_amount, shift_amount != masked_amount) }; let shift_amount = u32::try_from(shift_amount).unwrap(); // we masked so this will always fit // Compute the shifted result. - let result = if left_layout.abi.is_signed() { - let l = self.sign_extend(l, left_layout) as i128; + let result = if left.layout.abi.is_signed() { + let l = l_signed(); let result = match bin_op { Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(), Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(), _ => bug!(), }; - result as u128 + ScalarInt::truncate_from_int(result, left.layout.size).0 } else { - match bin_op { + let l = l_unsigned(); + let result = match bin_op { Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(), Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(), _ => bug!(), - } + }; + ScalarInt::truncate_from_uint(result, left.layout.size).0 }; - let truncated = self.truncate(result, left_layout); if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { throw_ub_custom!( fluent::const_eval_overflow_shift, - val = if right_layout.abi.is_signed() { - (self.sign_extend(r, right_layout) as i128).to_string() + val = if right.layout.abi.is_signed() { + r_signed().to_string() } else { - r.to_string() + r_unsigned().to_string() }, name = intrinsic_name ); } - return Ok((ImmTy::from_uint(truncated, left_layout), overflow)); + return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); } // For the remaining ops, the types must be the same on both sides - if left_layout.ty != right_layout.ty { + if left.layout.ty != right.layout.ty { span_bug!( self.cur_span(), "invalid asymmetric binary op {bin_op:?}: {l:?} ({l_ty}), {r:?} ({r_ty})", - l_ty = left_layout.ty, - r_ty = right_layout.ty, + l_ty = left.layout.ty, + r_ty = right.layout.ty, ) } - let size = left_layout.size; + let size = left.layout.size; // Operations that need special treatment for signed integers - if left_layout.abi.is_signed() { + if left.layout.abi.is_signed() { let op: Option<fn(&i128, &i128) -> bool> = match bin_op { Lt => Some(i128::lt), Le => Some(i128::le), @@ -236,18 +245,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => None, }; if let Some(op) = op { - let l = self.sign_extend(l, left_layout) as i128; - let r = self.sign_extend(r, right_layout) as i128; - return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false)); + return Ok((ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx), false)); } if bin_op == Cmp { - let l = self.sign_extend(l, left_layout) as i128; - let r = self.sign_extend(r, right_layout) as i128; - return Ok(self.three_way_compare(l, r)); + return Ok(self.three_way_compare(l_signed(), r_signed())); } let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op { - Div if r == 0 => throw_ub!(DivisionByZero), - Rem if r == 0 => throw_ub!(RemainderByZero), + Div if r.is_null() => throw_ub!(DivisionByZero), + Rem if r.is_null() => throw_ub!(RemainderByZero), Div => Some(i128::overflowing_div), Rem => Some(i128::overflowing_rem), Add | AddUnchecked => Some(i128::overflowing_add), @@ -256,8 +261,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { _ => None, }; if let Some(op) = op { - let l = self.sign_extend(l, left_layout) as i128; - let r = self.sign_extend(r, right_layout) as i128; + let l = l_signed(); + let r = r_signed(); // We need a special check for overflowing Rem and Div since they are *UB* // on overflow, which can happen with "int_min $OP -1". @@ -272,17 +277,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } let (result, oflo) = op(l, r); - // This may be out-of-bounds for the result type, so we have to truncate ourselves. + // This may be out-of-bounds for the result type, so we have to truncate. // If that truncation loses any information, we have an overflow. - let result = result as u128; - let truncated = self.truncate(result, left_layout); - let overflow = oflo || self.sign_extend(truncated, left_layout) != result; + let (result, lossy) = ScalarInt::truncate_from_int(result, left.layout.size); + let overflow = oflo || lossy; if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { throw_ub_custom!(fluent::const_eval_overflow, name = intrinsic_name); } - return Ok((ImmTy::from_uint(truncated, left_layout), overflow)); + return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); } } + // From here on it's okay to treat everything as unsigned. + let l = l_unsigned(); + let r = r_unsigned(); if bin_op == Cmp { return Ok(self.three_way_compare(l, r)); @@ -297,12 +304,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Gt => ImmTy::from_bool(l > r, *self.tcx), Ge => ImmTy::from_bool(l >= r, *self.tcx), - BitOr => ImmTy::from_uint(l | r, left_layout), - BitAnd => ImmTy::from_uint(l & r, left_layout), - BitXor => ImmTy::from_uint(l ^ r, left_layout), + BitOr => ImmTy::from_uint(l | r, left.layout), + BitAnd => ImmTy::from_uint(l & r, left.layout), + BitXor => ImmTy::from_uint(l ^ r, left.layout), Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Rem | Div => { - assert!(!left_layout.abi.is_signed()); + assert!(!left.layout.abi.is_signed()); let op: fn(u128, u128) -> (u128, bool) = match bin_op { Add | AddUnchecked => u128::overflowing_add, Sub | SubUnchecked => u128::overflowing_sub, @@ -316,21 +323,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let (result, oflo) = op(l, r); // Truncate to target type. // If that truncation loses any information, we have an overflow. - let truncated = self.truncate(result, left_layout); - let overflow = oflo || truncated != result; + let (result, lossy) = ScalarInt::truncate_from_uint(result, left.layout.size); + let overflow = oflo || lossy; if overflow && let Some(intrinsic_name) = throw_ub_on_overflow { throw_ub_custom!(fluent::const_eval_overflow, name = intrinsic_name); } - return Ok((ImmTy::from_uint(truncated, left_layout), overflow)); + return Ok((ImmTy::from_scalar_int(result, left.layout), overflow)); } _ => span_bug!( self.cur_span(), "invalid binary op {:?}: {:?}, {:?} (both {})", bin_op, - l, - r, - right_layout.ty, + left, + right, + right.layout.ty, ), }; @@ -427,9 +434,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { right.layout.ty ); - let l = left.to_scalar().to_bits(left.layout.size)?; - let r = right.to_scalar().to_bits(right.layout.size)?; - self.binary_int_op(bin_op, l, left.layout, r, right.layout) + self.binary_int_op(bin_op, left, right) } _ if left.layout.ty.is_any_ptr() => { // The RHS type must be a `pointer` *or an integer type* (for `Offset`). diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 1549eddabbc..8364a5a8d18 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -1058,7 +1058,7 @@ where } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs index 7a77f2c0dbb..78d05a6e195 100644 --- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs +++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs @@ -70,21 +70,21 @@ pub fn reverse_post_order<G: DirectedGraph + Successors>( } /// A "depth-first search" iterator for a directed graph. -pub struct DepthFirstSearch<'graph, G> +pub struct DepthFirstSearch<G> where - G: ?Sized + DirectedGraph + Successors, + G: DirectedGraph + Successors, { - graph: &'graph G, + graph: G, stack: Vec<G::Node>, visited: BitSet<G::Node>, } -impl<'graph, G> DepthFirstSearch<'graph, G> +impl<G> DepthFirstSearch<G> where - G: ?Sized + DirectedGraph + Successors, + G: DirectedGraph + Successors, { - pub fn new(graph: &'graph G) -> Self { - Self { graph, stack: vec![], visited: BitSet::new_empty(graph.num_nodes()) } + pub fn new(graph: G) -> Self { + Self { stack: vec![], visited: BitSet::new_empty(graph.num_nodes()), graph } } /// Version of `push_start_node` that is convenient for chained @@ -125,9 +125,9 @@ where } } -impl<G> std::fmt::Debug for DepthFirstSearch<'_, G> +impl<G> std::fmt::Debug for DepthFirstSearch<G> where - G: ?Sized + DirectedGraph + Successors, + G: DirectedGraph + Successors, { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut f = fmt.debug_set(); @@ -138,9 +138,9 @@ where } } -impl<G> Iterator for DepthFirstSearch<'_, G> +impl<G> Iterator for DepthFirstSearch<G> where - G: ?Sized + DirectedGraph + Successors, + G: DirectedGraph + Successors, { type Item = G::Node; diff --git a/compiler/rustc_data_structures/src/graph/mod.rs b/compiler/rustc_data_structures/src/graph/mod.rs index 3ae3023a91b..103ddd917bf 100644 --- a/compiler/rustc_data_structures/src/graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/mod.rs @@ -46,9 +46,35 @@ where .is_some() } -pub fn depth_first_search<G>(graph: &G, from: G::Node) -> iterate::DepthFirstSearch<'_, G> +pub fn depth_first_search<G>(graph: G, from: G::Node) -> iterate::DepthFirstSearch<G> where - G: ?Sized + Successors, + G: Successors, { iterate::DepthFirstSearch::new(graph).with_start_node(from) } + +pub fn depth_first_search_as_undirected<G>( + graph: G, + from: G::Node, +) -> iterate::DepthFirstSearch<impl Successors<Node = G::Node>> +where + G: Successors + Predecessors, +{ + struct AsUndirected<G>(G); + + impl<G: DirectedGraph> DirectedGraph for AsUndirected<G> { + type Node = G::Node; + + fn num_nodes(&self) -> usize { + self.0.num_nodes() + } + } + + impl<G: Successors + Predecessors> Successors for AsUndirected<G> { + fn successors(&self, node: Self::Node) -> impl Iterator<Item = Self::Node> { + self.0.successors(node).chain(self.0.predecessors(node)) + } + } + + iterate::DepthFirstSearch::new(AsUndirected(graph)).with_start_node(from) +} diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs index 26c86469fad..120244c8918 100644 --- a/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs +++ b/compiler/rustc_data_structures/src/graph/vec_graph/mod.rs @@ -1,99 +1,235 @@ -use crate::graph::{DirectedGraph, NumEdges, Successors}; +use crate::graph::{DirectedGraph, NumEdges, Predecessors, Successors}; use rustc_index::{Idx, IndexVec}; #[cfg(test)] mod tests; -pub struct VecGraph<N: Idx> { - /// Maps from a given node to an index where the set of successors - /// for that node starts. The index indexes into the `edges` - /// vector. To find the range for a given node, we look up the - /// start for that node and then the start for the next node - /// (i.e., with an index 1 higher) and get the range between the - /// two. This vector always has an extra entry so that this works - /// even for the max element. +/// A directed graph, efficient for cases where node indices are pre-existing. +/// +/// If `BR` is true, the graph will store back-references, allowing you to get predecessors. +pub struct VecGraph<N: Idx, const BR: bool = false> { + // This is basically a `HashMap<N, (Vec<N>, If<BR, Vec<N>>)>` -- a map from a node index, to + // a list of targets of outgoing edges and (if enabled) a list of sources of incoming edges. + // + // However, it is condensed into two arrays as an optimization. + // + // `node_starts[n]` is the start of the list of targets of outgoing edges for node `n`. + // So you can get node's successors with `edge_targets[node_starts[n]..node_starts[n + 1]]`. + // + // If `BR` is true (back references are enabled), then `node_starts[n + edge_count]` is the + // start of the list of *sources* of incoming edges. You can get predecessors of a node + // similarly to its successors but offsetting by `edge_count`. `edge_count` is + // `edge_targets.len()/2` (again, in case BR is true) because half of the vec is back refs. + // + // All of this might be confusing, so here is an example graph and its representation: + // + // n3 ----+ + // ^ | (if BR = true) + // | v outgoing edges incoming edges + // n0 -> n1 -> n2 ______________ __________________ + // / \ / \ + // node indices[1]: n0, n1, n2, n3, n0, n1, n2, n3, n/a + // vec indices: n0, n1, n2, n3, n4, n5, n6, n7, n8 + // node_starts: [0, 1, 3, 4 4, 4, 5, 7, 8] + // | | | | | | | | | + // | | +---+ +---+ | +---+ | + // | | | | | | | + // v v v v v v v + // edge_targets: [n1, n2, n3, n2 n0, n1, n3, n1] + // / \____/ | | \____/ \ + // n0->n1 / | | \ n3<-n1 + // / n3->n2 [2] n1<-n0 [2] \ + // n1->n2, n1->n3 n2<-n1, n2<-n3 + // + // The incoming edges are basically stored in the same way as outgoing edges, but offset and + // the graph they store is the inverse of the original. Last index in the `node_starts` array + // always points to one-past-the-end, so that we don't need to bound check `node_starts[n + 1]` + // + // [1]: "node indices" are the indices a user of `VecGraph` might use, + // note that they are different from "vec indices", + // which are the real indices you need to index `node_starts` + // + // [2]: Note that even though n2 also points to here, + // the next index also points here, so n2 has no + // successors (`edge_targets[3..3] = []`). + // Similarly with n0 and incoming edges + // + // If this is still confusing... then sorry :( + // + /// Indices into `edge_targets` that signify a start of list of edges. node_starts: IndexVec<N, usize>, + /// Targets (or sources for back refs) of edges edge_targets: Vec<N>, } -impl<N: Idx + Ord> VecGraph<N> { +impl<N: Idx + Ord, const BR: bool> VecGraph<N, BR> { pub fn new(num_nodes: usize, mut edge_pairs: Vec<(N, N)>) -> Self { + let num_edges = edge_pairs.len(); + + let nodes_cap = match BR { + // +1 for special entry at the end, pointing one past the end of `edge_targets` + false => num_nodes + 1, + // *2 for back references + true => (num_nodes * 2) + 1, + }; + + let edges_cap = match BR { + false => num_edges, + // *2 for back references + true => num_edges * 2, + }; + + let mut node_starts = IndexVec::with_capacity(nodes_cap); + let mut edge_targets = Vec::with_capacity(edges_cap); + // Sort the edges by the source -- this is important. edge_pairs.sort(); - let num_edges = edge_pairs.len(); + // Fill forward references + create_index( + num_nodes, + &mut edge_pairs.iter().map(|&(src, _)| src), + &mut edge_pairs.iter().map(|&(_, tgt)| tgt), + &mut edge_targets, + &mut node_starts, + ); - // Store the *target* of each edge into `edge_targets`. - let edge_targets: Vec<N> = edge_pairs.iter().map(|&(_, target)| target).collect(); - - // Create the *edge starts* array. We are iterating over the - // (sorted) edge pairs. We maintain the invariant that the - // length of the `node_starts` array is enough to store the - // current source node -- so when we see that the source node - // for an edge is greater than the current length, we grow the - // edge-starts array by just enough. - let mut node_starts = IndexVec::with_capacity(num_edges); - for (index, &(source, _)) in edge_pairs.iter().enumerate() { - // If we have a list like `[(0, x), (2, y)]`: - // - // - Start out with `node_starts` of `[]` - // - Iterate to `(0, x)` at index 0: - // - Push one entry because `node_starts.len()` (0) is <= the source (0) - // - Leaving us with `node_starts` of `[0]` - // - Iterate to `(2, y)` at index 1: - // - Push one entry because `node_starts.len()` (1) is <= the source (2) - // - Push one entry because `node_starts.len()` (2) is <= the source (2) - // - Leaving us with `node_starts` of `[0, 1, 1]` - // - Loop terminates - while node_starts.len() <= source.index() { - node_starts.push(index); - } - } + // Fill back references + if BR { + // Pop the special "last" entry, it will be replaced by first back ref + node_starts.pop(); - // Pad out the `node_starts` array so that it has `num_nodes + - // 1` entries. Continuing our example above, if `num_nodes` is - // be `3`, we would push one more index: `[0, 1, 1, 2]`. - // - // Interpretation of that vector: - // - // [0, 1, 1, 2] - // ---- range for N=2 - // ---- range for N=1 - // ---- range for N=0 - while node_starts.len() <= num_nodes { - node_starts.push(edge_targets.len()); - } + // Re-sort the edges so that they are sorted by target + edge_pairs.sort_by_key(|&(src, tgt)| (tgt, src)); - assert_eq!(node_starts.len(), num_nodes + 1); + create_index( + // Back essentially double the number of nodes + num_nodes * 2, + // NB: the source/target are switched here too + // NB: we double the key index, so that we can later use *2 to get the back references + &mut edge_pairs.iter().map(|&(_, tgt)| N::new(tgt.index() + num_nodes)), + &mut edge_pairs.iter().map(|&(src, _)| src), + &mut edge_targets, + &mut node_starts, + ); + } Self { node_starts, edge_targets } } /// Gets the successors for `source` as a slice. pub fn successors(&self, source: N) -> &[N] { + assert!(source.index() < self.num_nodes()); + let start_index = self.node_starts[source]; let end_index = self.node_starts[source.plus(1)]; &self.edge_targets[start_index..end_index] } } -impl<N: Idx> DirectedGraph for VecGraph<N> { +impl<N: Idx + Ord> VecGraph<N, true> { + /// Gets the predecessors for `target` as a slice. + pub fn predecessors(&self, target: N) -> &[N] { + assert!(target.index() < self.num_nodes()); + + let target = N::new(target.index() + self.num_nodes()); + + let start_index = self.node_starts[target]; + let end_index = self.node_starts[target.plus(1)]; + &self.edge_targets[start_index..end_index] + } +} + +/// Creates/initializes the index for the [`VecGraph`]. A helper for [`VecGraph::new`]. +/// +/// - `num_nodes` is the target number of nodes in the graph +/// - `sorted_edge_sources` are the edge sources, sorted +/// - `associated_edge_targets` are the edge *targets* in the same order as sources +/// - `edge_targets` is the vec of targets to be extended +/// - `node_starts` is the index to be filled +fn create_index<N: Idx + Ord>( + num_nodes: usize, + sorted_edge_sources: &mut dyn Iterator<Item = N>, + associated_edge_targets: &mut dyn Iterator<Item = N>, + edge_targets: &mut Vec<N>, + node_starts: &mut IndexVec<N, usize>, +) { + let offset = edge_targets.len(); + + // Store the *target* of each edge into `edge_targets`. + edge_targets.extend(associated_edge_targets); + + // Create the *edge starts* array. We are iterating over the + // (sorted) edge pairs. We maintain the invariant that the + // length of the `node_starts` array is enough to store the + // current source node -- so when we see that the source node + // for an edge is greater than the current length, we grow the + // edge-starts array by just enough. + for (index, source) in sorted_edge_sources.enumerate() { + // If we have a list like `[(0, x), (2, y)]`: + // + // - Start out with `node_starts` of `[]` + // - Iterate to `(0, x)` at index 0: + // - Push one entry because `node_starts.len()` (0) is <= the source (0) + // - Leaving us with `node_starts` of `[0]` + // - Iterate to `(2, y)` at index 1: + // - Push one entry because `node_starts.len()` (1) is <= the source (2) + // - Push one entry because `node_starts.len()` (2) is <= the source (2) + // - Leaving us with `node_starts` of `[0, 1, 1]` + // - Loop terminates + while node_starts.len() <= source.index() { + node_starts.push(index + offset); + } + } + + // Pad out the `node_starts` array so that it has `num_nodes + + // 1` entries. Continuing our example above, if `num_nodes` is + // be `3`, we would push one more index: `[0, 1, 1, 2]`. + // + // Interpretation of that vector: + // + // [0, 1, 1, 2] + // ---- range for N=2 + // ---- range for N=1 + // ---- range for N=0 + while node_starts.len() <= num_nodes { + node_starts.push(edge_targets.len()); + } + + assert_eq!(node_starts.len(), num_nodes + 1); +} + +impl<N: Idx, const BR: bool> DirectedGraph for VecGraph<N, BR> { type Node = N; fn num_nodes(&self) -> usize { - self.node_starts.len() - 1 + match BR { + false => self.node_starts.len() - 1, + // If back refs are enabled, half of the array is said back refs + true => (self.node_starts.len() - 1) / 2, + } } } -impl<N: Idx> NumEdges for VecGraph<N> { +impl<N: Idx, const BR: bool> NumEdges for VecGraph<N, BR> { fn num_edges(&self) -> usize { - self.edge_targets.len() + match BR { + false => self.edge_targets.len(), + // If back refs are enabled, half of the array is reversed edges for them + true => self.edge_targets.len() / 2, + } } } -impl<N: Idx + Ord> Successors for VecGraph<N> { +impl<N: Idx + Ord, const BR: bool> Successors for VecGraph<N, BR> { fn successors(&self, node: N) -> impl Iterator<Item = Self::Node> { self.successors(node).iter().cloned() } } + +impl<N: Idx + Ord> Predecessors for VecGraph<N, true> { + fn predecessors(&self, node: Self::Node) -> impl Iterator<Item = Self::Node> { + self.predecessors(node).iter().cloned() + } +} diff --git a/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs index 87c8d25f094..a077d9d0813 100644 --- a/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs +++ b/compiler/rustc_data_structures/src/graph/vec_graph/tests.rs @@ -18,10 +18,18 @@ fn create_graph() -> VecGraph<usize> { VecGraph::new(7, vec![(0, 1), (1, 2), (1, 3), (3, 4), (5, 1)]) } +fn create_graph_with_back_refs() -> VecGraph<usize, true> { + // Same as above + VecGraph::new(7, vec![(0, 1), (1, 2), (1, 3), (3, 4), (5, 1)]) +} + #[test] fn num_nodes() { let graph = create_graph(); assert_eq!(graph.num_nodes(), 7); + + let graph = create_graph_with_back_refs(); + assert_eq!(graph.num_nodes(), 7); } #[test] @@ -34,6 +42,27 @@ fn successors() { assert_eq!(graph.successors(4), &[] as &[usize]); assert_eq!(graph.successors(5), &[1]); assert_eq!(graph.successors(6), &[] as &[usize]); + + let graph = create_graph_with_back_refs(); + assert_eq!(graph.successors(0), &[1]); + assert_eq!(graph.successors(1), &[2, 3]); + assert_eq!(graph.successors(2), &[] as &[usize]); + assert_eq!(graph.successors(3), &[4]); + assert_eq!(graph.successors(4), &[] as &[usize]); + assert_eq!(graph.successors(5), &[1]); + assert_eq!(graph.successors(6), &[] as &[usize]); +} + +#[test] +fn predecessors() { + let graph = create_graph_with_back_refs(); + assert_eq!(graph.predecessors(0), &[]); + assert_eq!(graph.predecessors(1), &[0, 5]); + assert_eq!(graph.predecessors(2), &[1]); + assert_eq!(graph.predecessors(3), &[1]); + assert_eq!(graph.predecessors(4), &[3]); + assert_eq!(graph.predecessors(5), &[]); + assert_eq!(graph.predecessors(6), &[]); } #[test] @@ -41,4 +70,8 @@ fn dfs() { let graph = create_graph(); let dfs: Vec<_> = graph::depth_first_search(&graph, 0).collect(); assert_eq!(dfs, vec![0, 1, 3, 4, 2]); + + let graph = create_graph_with_back_refs(); + let dfs: Vec<_> = graph::depth_first_search(&graph, 0).collect(); + assert_eq!(dfs, vec![0, 1, 3, 4, 2]); } diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs index 5d345e788e9..a17e0da47a5 100644 --- a/compiler/rustc_errors/src/diagnostic.rs +++ b/compiler/rustc_errors/src/diagnostic.rs @@ -730,7 +730,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> { } } #[rustc_lint_diagnostics] - fn highlighted_note(&mut self, msg: Vec<StringPart>) -> &mut Self { + pub fn highlighted_note(&mut self, msg: Vec<StringPart>) -> &mut Self { self.sub_with_highlights(Level::Note, msg, MultiSpan::new()); self } diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index b4107bd4a2b..8d6b22a9fa9 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -102,9 +102,9 @@ pub type PResult<'a, T> = Result<T, PErr<'a>>; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } // `PResult` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(PResult<'_, ()>, 16); -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16); #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)] diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 9fff00ffeae..ffb50f4c92e 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -266,7 +266,7 @@ struct MatcherPos { } // This type is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(MatcherPos, 16); impl MatcherPos { diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 1ca30a48a51..e4f8d77dbc2 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -3786,7 +3786,7 @@ impl<'hir> Node<'hir> { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; // tidy-alphabetical-start diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index 3881e240ced..36553591de8 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -492,6 +492,7 @@ fn check_opaque_precise_captures<'tcx>(tcx: TyCtxt<'tcx>, opaque_def_id: LocalDe }; let mut expected_captures = UnordSet::default(); + let mut shadowed_captures = UnordSet::default(); let mut seen_params = UnordMap::default(); let mut prev_non_lifetime_param = None; for arg in precise_capturing_args { @@ -530,6 +531,21 @@ fn check_opaque_precise_captures<'tcx>(tcx: TyCtxt<'tcx>, opaque_def_id: LocalDe match tcx.named_bound_var(hir_id) { Some(ResolvedArg::EarlyBound(def_id)) => { expected_captures.insert(def_id); + + // Make sure we allow capturing these lifetimes through `Self` and + // `T::Assoc` projection syntax, too. These will occur when we only + // see lifetimes are captured after hir-lowering -- this aligns with + // the cases that were stabilized with the `impl_trait_projection` + // feature -- see <https://github.com/rust-lang/rust/pull/115659>. + if let DefKind::LifetimeParam = tcx.def_kind(def_id) + && let ty::ReEarlyParam(ty::EarlyParamRegion { def_id, .. }) + | ty::ReLateParam(ty::LateParamRegion { + bound_region: ty::BoundRegionKind::BrNamed(def_id, _), + .. + }) = *tcx.map_opaque_lifetime_to_parent_lifetime(def_id.expect_local()) + { + shadowed_captures.insert(def_id); + } } _ => { tcx.dcx().span_delayed_bug( @@ -555,23 +571,30 @@ fn check_opaque_precise_captures<'tcx>(tcx: TyCtxt<'tcx>, opaque_def_id: LocalDe ); continue; } + // If a param is shadowed by a early-bound (duplicated) lifetime, then + // it may or may not be captured as invariant, depending on if it shows + // up through `Self` or `T::Assoc` syntax. + if shadowed_captures.contains(¶m.def_id) { + continue; + } match param.kind { ty::GenericParamDefKind::Lifetime => { // Check if the lifetime param was captured but isn't named in the precise captures list. if variances[param.index as usize] == ty::Invariant { - let param_span = - if let ty::ReEarlyParam(ty::EarlyParamRegion { def_id, .. }) + let param_span = if let DefKind::OpaqueTy = + tcx.def_kind(tcx.parent(param.def_id)) + && let ty::ReEarlyParam(ty::EarlyParamRegion { def_id, .. }) | ty::ReLateParam(ty::LateParamRegion { bound_region: ty::BoundRegionKind::BrNamed(def_id, _), .. }) = *tcx .map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local()) - { - Some(tcx.def_span(def_id)) - } else { - None - }; + { + Some(tcx.def_span(def_id)) + } else { + None + }; // FIXME(precise_capturing): Structured suggestion for this would be useful tcx.dcx().emit_err(errors::LifetimeNotCaptured { use_span: tcx.def_span(param.def_id), diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs index 4d6a02f50bf..f83ddc51c76 100644 --- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs @@ -195,16 +195,19 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics { } Some(fn_def_id.to_def_id()) } - ItemKind::OpaqueTy(hir::OpaqueTy { - origin: hir::OpaqueTyOrigin::TyAlias { .. }, + ItemKind::OpaqueTy(&hir::OpaqueTy { + origin: hir::OpaqueTyOrigin::TyAlias { parent, in_assoc_ty }, .. }) => { - let parent_id = tcx.hir().get_parent_item(hir_id); - assert_ne!(parent_id, hir::CRATE_OWNER_ID); - debug!("generics_of: parent of opaque ty {:?} is {:?}", def_id, parent_id); + if in_assoc_ty { + assert!(matches!(tcx.def_kind(parent), DefKind::AssocTy)); + } else { + assert!(matches!(tcx.def_kind(parent), DefKind::TyAlias)); + } + debug!("generics_of: parent of opaque ty {:?} is {:?}", def_id, parent); // Opaque types are always nested within another item, and // inherit the generics of the item. - Some(parent_id.to_def_id()) + Some(parent.to_def_id()) } _ => None, }, diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs index 786754ed12f..bd2454f6368 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs @@ -279,13 +279,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } Entry::Occupied(mut entry) => { debug!(" - composing on top of {:?}", entry.get()); - match (&entry.get()[..], &adj[..]) { - // Applying any adjustment on top of a NeverToAny - // is a valid NeverToAny adjustment, because it can't - // be reached. - (&[Adjustment { kind: Adjust::NeverToAny, .. }], _) => return, + match (&mut entry.get_mut()[..], &adj[..]) { ( - &[ + [Adjustment { kind: Adjust::NeverToAny, target }], + &[.., Adjustment { target: new_target, .. }], + ) => { + // NeverToAny coercion can target any type, so instead of adding a new + // adjustment on top we can change the target. + // + // This is required for things like `a == a` (where `a: !`) to produce + // valid MIR -- we need borrow adjustment from things like `==` to change + // the type to `&!` (or `&()` depending on the fallback). This might be + // relevant even in unreachable code. + *target = new_target; + } + + ( + &mut [ Adjustment { kind: Adjust::Deref(_), .. }, Adjustment { kind: Adjust::Borrow(AutoBorrow::Ref(..)), .. }, ], @@ -294,11 +304,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { .., // Any following adjustments are allowed. ], ) => { - // A reborrow has no effect before a dereference. + // A reborrow has no effect before a dereference, so we can safely replace adjustments. + *entry.get_mut() = adj; } - // FIXME: currently we never try to compose autoderefs - // and ReifyFnPointer/UnsafeFnPointer, but we could. + _ => { + // FIXME: currently we never try to compose autoderefs + // and ReifyFnPointer/UnsafeFnPointer, but we could. self.dcx().span_delayed_bug( expr.span, format!( @@ -308,9 +320,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { adj ), ); + + *entry.get_mut() = adj; } } - *entry.get_mut() = adj; } } diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index c7e563035fc..e37af6610dd 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -400,7 +400,7 @@ enum Chunk { } // This type is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] crate::static_assert_size!(Chunk, 16); impl<T> ChunkedBitSet<T> { diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs index 29c9f08a166..fdae9d84b5f 100644 --- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs +++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs @@ -61,7 +61,7 @@ use crate::traits::{ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_errors::{ codes::*, pluralize, struct_span_code_err, Applicability, Diag, DiagCtxt, DiagStyledString, - ErrorGuaranteed, IntoDiagArg, + ErrorGuaranteed, IntoDiagArg, StringPart, }; use rustc_hir as hir; use rustc_hir::def::DefKind; @@ -1917,6 +1917,23 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> { ); if !is_simple_error || terr.must_include_note() { diag.note_expected_found(&expected_label, expected, &found_label, found); + + if let Some(ty::Closure(_, args)) = + exp_found.map(|expected_type_found| expected_type_found.found.kind()) + { + diag.highlighted_note(vec![ + StringPart::normal("closure has signature: `"), + StringPart::highlighted( + self.tcx + .signature_unclosure( + args.as_closure().sig(), + rustc_hir::Unsafety::Normal, + ) + .to_string(), + ), + StringPart::normal("`"), + ]); + } } } } diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index 255e688fbc1..355fcf59495 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -477,7 +477,7 @@ pub enum SubregionOrigin<'tcx> { } // `SubregionOrigin` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(SubregionOrigin<'_>, 32); impl<'tcx> SubregionOrigin<'tcx> { diff --git a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs index 06f8dd4a4c6..6e8efa3e7c1 100644 --- a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs +++ b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs @@ -382,7 +382,7 @@ impl<'tcx> MiniGraph<'tcx> { edges.push((source_node, target_node)); }, ); - let graph = VecGraph::new(nodes.len(), edges); + let graph = VecGraph::<_, false>::new(nodes.len(), edges); let sccs = Sccs::new(&graph); Self { nodes, sccs } } diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs index 0444cbe2ee4..a5f52420a84 100644 --- a/compiler/rustc_infer/src/lib.rs +++ b/compiler/rustc_infer/src/lib.rs @@ -32,7 +32,7 @@ #[macro_use] extern crate rustc_macros; -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] #[macro_use] extern crate rustc_data_structures; #[macro_use] diff --git a/compiler/rustc_infer/src/traits/mod.rs b/compiler/rustc_infer/src/traits/mod.rs index 94ad0f5b1c8..7b4e085293c 100644 --- a/compiler/rustc_infer/src/traits/mod.rs +++ b/compiler/rustc_infer/src/traits/mod.rs @@ -112,7 +112,7 @@ impl<'tcx> PolyTraitObligation<'tcx> { } // `PredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(PredicateObligation<'_>, 48); pub type PredicateObligations<'tcx> = Vec<PredicateObligation<'tcx>>; diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index e563728c893..8d741ef4c1b 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -19,7 +19,9 @@ use rustc_span::edition::{Edition, DEFAULT_EDITION}; use rustc_span::source_map::{RealFileLoader, SourceMapInputs}; use rustc_span::symbol::sym; use rustc_span::{FileName, SourceFileHashAlgorithm}; -use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel}; +use rustc_target::spec::{ + CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel, WasmCAbi, +}; use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel}; use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZero; @@ -758,7 +760,7 @@ fn test_unstable_options_tracking_hash() { ); tracked!(codegen_backend, Some("abc".to_string())); tracked!(collapse_macro_debuginfo, CollapseMacroDebuginfo::Yes); - tracked!(coverage_options, CoverageOptions { branch: true }); + tracked!(coverage_options, CoverageOptions { branch: true, mcdc: true }); tracked!(crate_attr, vec!["abc".to_string()]); tracked!(cross_crate_inline_threshold, InliningThreshold::Always); tracked!(debug_info_for_profiling, true); @@ -851,6 +853,7 @@ fn test_unstable_options_tracking_hash() { tracked!(verify_llvm_ir, true); tracked!(virtual_function_elimination, true); tracked!(wasi_exec_model, Some(WasiExecModel::Reactor)); + tracked!(wasm_c_abi, WasmCAbi::Spec); // tidy-alphabetical-end macro_rules! tracked_no_crate_hash { diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index ca84e930c24..83fff98bad5 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -88,6 +88,10 @@ pub enum TokenKind { /// tokens. UnknownPrefix, + /// Similar to the above, but *always* an error on every edition. This is used + /// for emoji identifier recovery, as those are not meant to be ever accepted. + InvalidPrefix, + /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid /// suffix, but may be present here on string and float literals. Users of /// this type will need to check for and reject that case. @@ -528,7 +532,7 @@ impl Cursor<'_> { // Known prefixes must have been handled earlier. So if // we see a prefix here, it is definitely an unknown prefix. match self.first() { - '#' | '"' | '\'' => UnknownPrefix, + '#' | '"' | '\'' => InvalidPrefix, _ => InvalidIdent, } } diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp index 64e6c18092f..8871f410e36 100644 --- a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp @@ -175,7 +175,7 @@ extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { extern "C" LLVMRustResult LLVMRustWriteArchive(char *Dst, size_t NumMembers, const LLVMRustArchiveMemberRef *NewMembers, - bool WriteSymbtab, LLVMRustArchiveKind RustKind) { + bool WriteSymbtab, LLVMRustArchiveKind RustKind, bool isEC) { std::vector<NewArchiveMember> Members; auto Kind = fromRust(RustKind); @@ -207,7 +207,7 @@ LLVMRustWriteArchive(char *Dst, size_t NumMembers, auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); #else auto SymtabMode = WriteSymbtab ? SymtabWritingMode::NormalSymtab : SymtabWritingMode::NoSymtab; - auto Result = writeArchive(Dst, Members, SymtabMode, Kind, true, false); + auto Result = writeArchive(Dst, Members, SymtabMode, Kind, true, false, nullptr, isEC); #endif if (!Result) return LLVMRustResult::Success; diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 0998b463a88..e842f47f48c 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -4,8 +4,6 @@ #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" #include "llvm/ProfileData/InstrProf.h" -#include <iostream> - using namespace llvm; // FFI equivalent of enum `llvm::coverage::Counter::CounterKind` @@ -43,6 +41,8 @@ enum class LLVMRustCounterMappingRegionKind { SkippedRegion = 2, GapRegion = 3, BranchRegion = 4, + MCDCDecisionRegion = 5, + MCDCBranchRegion = 6 }; static coverage::CounterMappingRegion::RegionKind @@ -58,15 +58,102 @@ fromRust(LLVMRustCounterMappingRegionKind Kind) { return coverage::CounterMappingRegion::GapRegion; case LLVMRustCounterMappingRegionKind::BranchRegion: return coverage::CounterMappingRegion::BranchRegion; +#if LLVM_VERSION_GE(18, 0) + case LLVMRustCounterMappingRegionKind::MCDCDecisionRegion: + return coverage::CounterMappingRegion::MCDCDecisionRegion; + case LLVMRustCounterMappingRegionKind::MCDCBranchRegion: + return coverage::CounterMappingRegion::MCDCBranchRegion; +#else + case LLVMRustCounterMappingRegionKind::MCDCDecisionRegion: + break; + case LLVMRustCounterMappingRegionKind::MCDCBranchRegion: + break; +#endif } report_fatal_error("Bad LLVMRustCounterMappingRegionKind!"); } +enum LLVMRustMCDCParametersTag { + None = 0, + Decision = 1, + Branch = 2, +}; + +struct LLVMRustMCDCDecisionParameters { + uint32_t BitmapIdx; + uint16_t NumConditions; +}; + +struct LLVMRustMCDCBranchParameters { + int16_t ConditionID; + int16_t ConditionIDs[2]; +}; + +struct LLVMRustMCDCParameters { + LLVMRustMCDCParametersTag Tag; + LLVMRustMCDCDecisionParameters DecisionParameters; + LLVMRustMCDCBranchParameters BranchParameters; +}; + +// LLVM representations for `MCDCParameters` evolved from LLVM 18 to 19. +// Look at representations in 18 +// https://github.com/rust-lang/llvm-project/blob/66a2881a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L253-L263 +// and representations in 19 +// https://github.com/llvm/llvm-project/blob/843cc474faefad1d639f4c44c1cf3ad7dbda76c8/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h +#if LLVM_VERSION_GE(18, 0) && LLVM_VERSION_LT(19, 0) +static coverage::CounterMappingRegion::MCDCParameters +fromRust(LLVMRustMCDCParameters Params) { + auto parameter = coverage::CounterMappingRegion::MCDCParameters{}; + switch (Params.Tag) { + case LLVMRustMCDCParametersTag::None: + return parameter; + case LLVMRustMCDCParametersTag::Decision: + parameter.BitmapIdx = + static_cast<unsigned>(Params.DecisionParameters.BitmapIdx), + parameter.NumConditions = + static_cast<unsigned>(Params.DecisionParameters.NumConditions); + return parameter; + case LLVMRustMCDCParametersTag::Branch: + parameter.ID = static_cast<coverage::CounterMappingRegion::MCDCConditionID>( + Params.BranchParameters.ConditionID), + parameter.FalseID = + static_cast<coverage::CounterMappingRegion::MCDCConditionID>( + Params.BranchParameters.ConditionIDs[0]), + parameter.TrueID = + static_cast<coverage::CounterMappingRegion::MCDCConditionID>( + Params.BranchParameters.ConditionIDs[1]); + return parameter; + } + report_fatal_error("Bad LLVMRustMCDCParametersTag!"); +} +#elif LLVM_VERSION_GE(19, 0) +static coverage::mcdc::Parameters fromRust(LLVMRustMCDCParameters Params) { + switch (Params.Tag) { + case LLVMRustMCDCParametersTag::None: + return std::monostate(); + case LLVMRustMCDCParametersTag::Decision: + return coverage::mcdc::DecisionParameters( + Params.DecisionParameters.BitmapIdx, + Params.DecisionParameters.NumConditions); + case LLVMRustMCDCParametersTag::Branch: + return coverage::mcdc::BranchParameters( + static_cast<coverage::mcdc::ConditionID>( + Params.BranchParameters.ConditionID), + {static_cast<coverage::mcdc::ConditionID>( + Params.BranchParameters.ConditionIDs[0]), + static_cast<coverage::mcdc::ConditionID>( + Params.BranchParameters.ConditionIDs[1])}); + } + report_fatal_error("Bad LLVMRustMCDCParametersTag!"); +} +#endif + // FFI equivalent of struct `llvm::coverage::CounterMappingRegion` // https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304 struct LLVMRustCounterMappingRegion { LLVMRustCounter Count; LLVMRustCounter FalseCount; + LLVMRustMCDCParameters MCDCParameters; uint32_t FileID; uint32_t ExpandedFileID; uint32_t LineStart; @@ -135,7 +222,8 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer( MappingRegions.emplace_back( fromRust(Region.Count), fromRust(Region.FalseCount), #if LLVM_VERSION_GE(18, 0) && LLVM_VERSION_LT(19, 0) - coverage::CounterMappingRegion::MCDCParameters{}, + // LLVM 19 may move this argument to last. + fromRust(Region.MCDCParameters), #endif Region.FileID, Region.ExpandedFileID, // File IDs, then region info. Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 6e11fd629e4..565bdc3af03 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1522,6 +1522,7 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { delete Bundle; } +// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, OperandBundleDef **OpBundlesIndirect, @@ -1546,6 +1547,33 @@ extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) unwrap(M), llvm::Intrinsic::instrprof_increment)); } +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_condbitmap_update)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Src, unsigned SrcAlign, @@ -1574,6 +1602,7 @@ extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); } +// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). extern "C" LLVMValueRef LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, @@ -1596,6 +1625,7 @@ LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, Name)); } +// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). extern "C" LLVMValueRef LLVMRustBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMBasicBlockRef DefaultDest, diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs index 155af062012..596271c1e47 100644 --- a/compiler/rustc_middle/src/mir/consts.rs +++ b/compiler/rustc_middle/src/mir/consts.rs @@ -70,7 +70,7 @@ pub enum ConstValue<'tcx> { }, } -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(ConstValue<'_>, 24); impl<'tcx> ConstValue<'tcx> { @@ -87,7 +87,7 @@ impl<'tcx> ConstValue<'tcx> { } pub fn try_to_bits(&self, size: Size) -> Option<u128> { - self.try_to_scalar_int()?.to_bits(size).ok() + self.try_to_scalar_int()?.try_to_bits(size).ok() } pub fn try_to_bool(&self) -> Option<bool> { @@ -244,6 +244,8 @@ impl<'tcx> Const<'tcx> { Const::Ty(c) => match c.kind() { ty::ConstKind::Value(valtree) if c.ty().is_primitive() => { // A valtree of a type where leaves directly represent the scalar const value. + // Just checking whether it is a leaf is insufficient as e.g. references are leafs + // but the leaf value is the value they point to, not the reference itself! Some(valtree.unwrap_leaf().into()) } _ => None, @@ -255,12 +257,22 @@ impl<'tcx> Const<'tcx> { #[inline] pub fn try_to_scalar_int(self) -> Option<ScalarInt> { - self.try_to_scalar()?.try_to_int().ok() + // This is equivalent to `self.try_to_scalar()?.try_to_int().ok()`, but measurably faster. + match self { + Const::Val(ConstValue::Scalar(Scalar::Int(x)), _) => Some(x), + Const::Ty(c) => match c.kind() { + ty::ConstKind::Value(valtree) if c.ty().is_primitive() => { + Some(valtree.unwrap_leaf()) + } + _ => None, + }, + _ => None, + } } #[inline] pub fn try_to_bits(self, size: Size) -> Option<u128> { - self.try_to_scalar_int()?.to_bits(size).ok() + self.try_to_scalar_int()?.try_to_bits(size).ok() } #[inline] @@ -334,7 +346,7 @@ impl<'tcx> Const<'tcx> { let int = self.try_eval_scalar_int(tcx, param_env)?; let size = tcx.layout_of(param_env.with_reveal_all_normalized(tcx).and(self.ty())).ok()?.size; - int.to_bits(size).ok() + int.try_to_bits(size).ok() } /// Panics if the value cannot be evaluated or doesn't contain a valid integer of the given type. diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index 582a1806688..b1d0c815ae0 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -51,6 +51,25 @@ rustc_index::newtype_index! { pub struct ExpressionId {} } +rustc_index::newtype_index! { + /// ID of a mcdc condition. Used by llvm to check mcdc coverage. + /// + /// Note for future: the max limit of 0xFFFF is probably too loose. Actually llvm does not + /// support decisions with too many conditions (7 and more at LLVM 18 while may be hundreds at 19) + /// and represents it with `int16_t`. This max value may be changed once we could + /// figure out an accurate limit. + #[derive(HashStable)] + #[encodable] + #[orderable] + #[max = 0xFFFF] + #[debug_format = "ConditionId({})"] + pub struct ConditionId {} +} + +impl ConditionId { + pub const NONE: Self = Self::from_u32(0); +} + /// Enum that can hold a constant zero value, the ID of an physical coverage /// counter, or the ID of a coverage-counter expression. /// @@ -106,6 +125,22 @@ pub enum CoverageKind { /// mappings. Intermediate expressions with no direct mappings are /// retained/zeroed based on whether they are transitively used.) ExpressionUsed { id: ExpressionId }, + + /// Marks the point in MIR control flow represented by a evaluated condition. + /// + /// This is eventually lowered to `llvm.instrprof.mcdc.condbitmap.update` in LLVM IR. + /// + /// If this statement does not survive MIR optimizations, the condition would never be + /// taken as evaluated. + CondBitmapUpdate { id: ConditionId, value: bool }, + + /// Marks the point in MIR control flow represented by a evaluated decision. + /// + /// This is eventually lowered to `llvm.instrprof.mcdc.tvbitmap.update` in LLVM IR. + /// + /// If this statement does not survive MIR optimizations, the decision would never be + /// taken as evaluated. + TestVectorBitmapUpdate { bitmap_idx: u32 }, } impl Debug for CoverageKind { @@ -116,6 +151,12 @@ impl Debug for CoverageKind { BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()), CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()), ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()), + CondBitmapUpdate { id, value } => { + write!(fmt, "CondBitmapUpdate({:?}, {:?})", id.index(), value) + } + TestVectorBitmapUpdate { bitmap_idx } => { + write!(fmt, "TestVectorUpdate({:?})", bitmap_idx) + } } } } @@ -172,16 +213,23 @@ pub enum MappingKind { Code(CovTerm), /// Associates a branch region with separate counters for true and false. Branch { true_term: CovTerm, false_term: CovTerm }, + /// Associates a branch region with separate counters for true and false. + MCDCBranch { true_term: CovTerm, false_term: CovTerm, mcdc_params: ConditionInfo }, + /// Associates a decision region with a bitmap and number of conditions. + MCDCDecision(DecisionInfo), } impl MappingKind { /// Iterator over all coverage terms in this mapping kind. pub fn terms(&self) -> impl Iterator<Item = CovTerm> { - let one = |a| std::iter::once(a).chain(None); - let two = |a, b| std::iter::once(a).chain(Some(b)); + let zero = || None.into_iter().chain(None); + let one = |a| Some(a).into_iter().chain(None); + let two = |a, b| Some(a).into_iter().chain(Some(b)); match *self { Self::Code(term) => one(term), Self::Branch { true_term, false_term } => two(true_term, false_term), + Self::MCDCBranch { true_term, false_term, .. } => two(true_term, false_term), + Self::MCDCDecision(_) => zero(), } } @@ -193,6 +241,12 @@ impl MappingKind { Self::Branch { true_term, false_term } => { Self::Branch { true_term: map_fn(true_term), false_term: map_fn(false_term) } } + Self::MCDCBranch { true_term, false_term, mcdc_params } => Self::MCDCBranch { + true_term: map_fn(true_term), + false_term: map_fn(false_term), + mcdc_params, + }, + Self::MCDCDecision(param) => Self::MCDCDecision(param), } } } @@ -212,7 +266,7 @@ pub struct Mapping { pub struct FunctionCoverageInfo { pub function_source_hash: u64, pub num_counters: usize, - + pub mcdc_bitmap_bytes: u32, pub expressions: IndexVec<ExpressionId, Expression>, pub mappings: Vec<Mapping>, } @@ -226,6 +280,8 @@ pub struct BranchInfo { /// data structures without having to scan the entire body first. pub num_block_markers: usize, pub branch_spans: Vec<BranchSpan>, + pub mcdc_branch_spans: Vec<MCDCBranchSpan>, + pub mcdc_decision_spans: Vec<MCDCDecisionSpan>, } #[derive(Clone, Debug)] @@ -235,3 +291,45 @@ pub struct BranchSpan { pub true_marker: BlockMarkerId, pub false_marker: BlockMarkerId, } + +#[derive(Copy, Clone, Debug)] +#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] +pub struct ConditionInfo { + pub condition_id: ConditionId, + pub true_next_id: ConditionId, + pub false_next_id: ConditionId, +} + +impl Default for ConditionInfo { + fn default() -> Self { + Self { + condition_id: ConditionId::NONE, + true_next_id: ConditionId::NONE, + false_next_id: ConditionId::NONE, + } + } +} + +#[derive(Clone, Debug)] +#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] +pub struct MCDCBranchSpan { + pub span: Span, + pub condition_info: ConditionInfo, + pub true_marker: BlockMarkerId, + pub false_marker: BlockMarkerId, +} + +#[derive(Copy, Clone, Debug)] +#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] +pub struct DecisionInfo { + pub bitmap_idx: u32, + pub conditions_num: u16, +} + +#[derive(Clone, Debug)] +#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)] +pub struct MCDCDecisionSpan { + pub span: Span, + pub conditions_num: usize, + pub end_markers: Vec<BlockMarkerId>, +} diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index e9be26d058b..65ce1cd8f50 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -88,7 +88,7 @@ pub type EvalToConstValueResult<'tcx> = Result<ConstValue<'tcx>, ErrorHandled>; /// This is needed in `thir::pattern::lower_inline_const`. pub type EvalToValTreeResult<'tcx> = Result<Option<ValTree<'tcx>>, ErrorHandled>; -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(InterpErrorInfo<'_>, 8); /// Packages the kind of error we got from the const code interpreter diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index 9f9433e483b..9728967860a 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -37,7 +37,7 @@ pub enum Scalar<Prov = CtfeProvenance> { Ptr(Pointer<Prov>, u8), } -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(Scalar, 24); // We want the `Debug` output to be readable as it is used by `derive(Debug)` for @@ -236,7 +236,7 @@ impl<Prov> Scalar<Prov> { ) -> Result<Either<u128, Pointer<Prov>>, ScalarSizeMismatch> { assert_ne!(target_size.bytes(), 0, "you should never look at the bits of a ZST"); Ok(match self { - Scalar::Int(int) => Left(int.to_bits(target_size).map_err(|size| { + Scalar::Int(int) => Left(int.try_to_bits(target_size).map_err(|size| { ScalarSizeMismatch { target_size: target_size.bytes(), data_size: size.bytes() } })?), Scalar::Ptr(ptr, sz) => { @@ -301,6 +301,11 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> { } #[inline(always)] + pub fn to_scalar_int(self) -> InterpResult<'tcx, ScalarInt> { + self.try_to_int().map_err(|_| err_unsup!(ReadPointerAsInt(None)).into()) + } + + #[inline(always)] #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980) pub fn assert_int(self) -> ScalarInt { self.try_to_int().unwrap() @@ -311,16 +316,13 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> { #[inline] pub fn to_bits(self, target_size: Size) -> InterpResult<'tcx, u128> { assert_ne!(target_size.bytes(), 0, "you should never look at the bits of a ZST"); - self.try_to_int() - .map_err(|_| err_unsup!(ReadPointerAsInt(None)))? - .to_bits(target_size) - .map_err(|size| { - err_ub!(ScalarSizeMismatch(ScalarSizeMismatch { - target_size: target_size.bytes(), - data_size: size.bytes(), - })) - .into() - }) + self.to_scalar_int()?.try_to_bits(target_size).map_err(|size| { + err_ub!(ScalarSizeMismatch(ScalarSizeMismatch { + target_size: target_size.bytes(), + data_size: size.bytes(), + })) + .into() + }) } #[inline(always)] diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index c331df1054b..43e4e8216e1 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1813,7 +1813,7 @@ impl DefLocation { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 15bd5c08965..a3cdcec9ec0 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -475,7 +475,8 @@ fn write_coverage_branch_info( branch_info: &coverage::BranchInfo, w: &mut dyn io::Write, ) -> io::Result<()> { - let coverage::BranchInfo { branch_spans, .. } = branch_info; + let coverage::BranchInfo { branch_spans, mcdc_branch_spans, mcdc_decision_spans, .. } = + branch_info; for coverage::BranchSpan { span, true_marker, false_marker } in branch_spans { writeln!( @@ -483,7 +484,26 @@ fn write_coverage_branch_info( "{INDENT}coverage branch {{ true: {true_marker:?}, false: {false_marker:?} }} => {span:?}", )?; } - if !branch_spans.is_empty() { + + for coverage::MCDCBranchSpan { span, condition_info, true_marker, false_marker } in + mcdc_branch_spans + { + writeln!( + w, + "{INDENT}coverage mcdc branch {{ condition_id: {:?}, true: {true_marker:?}, false: {false_marker:?} }} => {span:?}", + condition_info.condition_id + )?; + } + + for coverage::MCDCDecisionSpan { span, conditions_num, end_markers } in mcdc_decision_spans { + writeln!( + w, + "{INDENT}coverage mcdc decision {{ conditions_num: {conditions_num:?}, end: {end_markers:?} }} => {span:?}" + )?; + } + + if !branch_spans.is_empty() || !mcdc_branch_spans.is_empty() || !mcdc_decision_spans.is_empty() + { writeln!(w)?; } diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 0f622127430..e3f58729fbd 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -213,7 +213,7 @@ pub struct ClosureOutlivesRequirement<'tcx> { } // Make sure this enum doesn't unintentionally grow -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16); /// Outlives-constraints can be categorized to determine whether and why they @@ -361,4 +361,8 @@ pub struct CoverageIdsInfo { /// InstrumentCoverage MIR pass, if the highest-numbered counter increments /// were removed by MIR optimizations. pub max_counter_id: mir::coverage::CounterId, + + /// Coverage codegen for mcdc needs to know the size of the global bitmap so that it can + /// set the `bytemap-bytes` argument of the `llvm.instrprof.mcdc.tvbitmap.update` intrinsic. + pub mcdc_bitmap_bytes: u32, } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 9b409574026..97c3eb55638 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -1453,7 +1453,7 @@ pub enum BinOp { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; // tidy-alphabetical-start diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index b86aa601ce8..506003ff7c0 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -14,7 +14,7 @@ pub struct PlaceTy<'tcx> { } // At least on 64 bit systems, `PlaceTy` should not be larger than two or three pointers. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(PlaceTy<'_>, 16); impl<'tcx> PlaceTy<'tcx> { diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 643861972c4..038d3fe93c4 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -322,7 +322,7 @@ macro_rules! define_callbacks { // Ensure that keys grow no larger than 72 bytes by accident. // Increase this limit if necessary, but do try to keep the size low if possible - #[cfg(all(any(target_arch = "x86_64", target_arch="aarch64"), target_pointer_width = "64"))] + #[cfg(target_pointer_width = "64")] const _: () = { if mem::size_of::<Key<'static>>() > 72 { panic!("{}", concat!( @@ -337,7 +337,7 @@ macro_rules! define_callbacks { // Ensure that values grow no larger than 64 bytes by accident. // Increase this limit if necessary, but do try to keep the size low if possible - #[cfg(all(any(target_arch = "x86_64", target_arch="aarch64"), target_pointer_width = "64"))] + #[cfg(target_pointer_width = "64")] const _: () = { if mem::size_of::<Value<'static>>() > 64 { panic!("{}", concat!( diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs index 3b705017710..d52b1efce4b 100644 --- a/compiler/rustc_middle/src/thir.rs +++ b/compiler/rustc_middle/src/thir.rs @@ -16,7 +16,7 @@ use rustc_hir::{BindingMode, ByRef, HirId, MatchSource, RangeEnd}; use rustc_index::newtype_index; use rustc_index::IndexVec; use rustc_middle::middle::region; -use rustc_middle::mir::interpret::{AllocId, Scalar}; +use rustc_middle::mir::interpret::AllocId; use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, UnOp}; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::IntegerExt; @@ -1006,18 +1006,20 @@ impl<'tcx> PatRangeBoundary<'tcx> { // This code is hot when compiling matches with many ranges. So we // special-case extraction of evaluated scalars for speed, for types where - // raw data comparisons are appropriate. E.g. `unicode-normalization` has + // we can do scalar comparisons. E.g. `unicode-normalization` has // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared // in this way. - (Finite(mir::Const::Ty(a)), Finite(mir::Const::Ty(b))) - if matches!(ty.kind(), ty::Uint(_) | ty::Char) => - { - return Some(a.to_valtree().cmp(&b.to_valtree())); + (Finite(a), Finite(b)) if matches!(ty.kind(), ty::Int(_) | ty::Uint(_) | ty::Char) => { + if let (Some(a), Some(b)) = (a.try_to_scalar_int(), b.try_to_scalar_int()) { + let sz = ty.primitive_size(tcx); + let cmp = match ty.kind() { + ty::Uint(_) | ty::Char => a.assert_uint(sz).cmp(&b.assert_uint(sz)), + ty::Int(_) => a.assert_int(sz).cmp(&b.assert_int(sz)), + _ => unreachable!(), + }; + return Some(cmp); + } } - ( - Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(a)), _)), - Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(b)), _)), - ) if matches!(ty.kind(), ty::Uint(_) | ty::Char) => return Some(a.cmp(&b)), _ => {} } @@ -1203,7 +1205,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> { } // Some nodes are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; // tidy-alphabetical-start diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs index 28f6184a34e..ceee3ea48e3 100644 --- a/compiler/rustc_middle/src/traits/mod.rs +++ b/compiler/rustc_middle/src/traits/mod.rs @@ -551,7 +551,7 @@ impl<'tcx> ObligationCauseCode<'tcx> { } // `ObligationCauseCode` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(ObligationCauseCode<'_>, 48); #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 49b806b8369..fd4573c1603 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -22,6 +22,9 @@ pub use valtree::*; pub type ConstKind<'tcx> = IrConstKind<TyCtxt<'tcx>>; +#[cfg(target_pointer_width = "64")] +static_assert_size!(ConstKind<'_>, 32); + /// Use this rather than `ConstData`, whenever possible. #[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable)] #[rustc_pass_by_value] @@ -59,7 +62,7 @@ pub struct ConstData<'tcx> { pub kind: ConstKind<'tcx>, } -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(ConstData<'_>, 40); impl<'tcx> Const<'tcx> { @@ -406,7 +409,7 @@ impl<'tcx> Const<'tcx> { let size = tcx.layout_of(param_env.with_reveal_all_normalized(tcx).and(self.ty())).ok()?.size; // if `ty` does not depend on generic parameters, use an empty param_env - int.to_bits(size).ok() + int.try_to_bits(size).ok() } #[inline] diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs index 71d7dfd8b01..40ac87873a0 100644 --- a/compiler/rustc_middle/src/ty/consts/int.rs +++ b/compiler/rustc_middle/src/ty/consts/int.rs @@ -126,7 +126,7 @@ impl IntoDiagArg for ConstInt { /// /// This is a packed struct in order to allow this type to be optimally embedded in enums /// (like Scalar). -#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[derive(Clone, Copy, Eq, PartialEq, Hash)] #[repr(packed)] pub struct ScalarInt { /// The first `size` bytes of `data` are the value. @@ -167,9 +167,12 @@ impl<D: Decoder> Decodable<D> for ScalarInt { impl ScalarInt { pub const TRUE: ScalarInt = ScalarInt { data: 1_u128, size: NonZero::new(1).unwrap() }; - pub const FALSE: ScalarInt = ScalarInt { data: 0_u128, size: NonZero::new(1).unwrap() }; + fn raw(data: u128, size: Size) -> Self { + Self { data, size: NonZero::new(size.bytes() as u8).unwrap() } + } + #[inline] pub fn size(self) -> Size { Size::from_bytes(self.size.get()) @@ -196,7 +199,7 @@ impl ScalarInt { #[inline] pub fn null(size: Size) -> Self { - Self { data: 0, size: NonZero::new(size.bytes() as u8).unwrap() } + Self::raw(0, size) } #[inline] @@ -207,11 +210,15 @@ impl ScalarInt { #[inline] pub fn try_from_uint(i: impl Into<u128>, size: Size) -> Option<Self> { let data = i.into(); - if size.truncate(data) == data { - Some(Self { data, size: NonZero::new(size.bytes() as u8).unwrap() }) - } else { - None - } + if size.truncate(data) == data { Some(Self::raw(data, size)) } else { None } + } + + /// Returns the truncated result, and whether truncation changed the value. + #[inline] + pub fn truncate_from_uint(i: impl Into<u128>, size: Size) -> (Self, bool) { + let data = i.into(); + let r = Self::raw(size.truncate(data), size); + (r, r.data != data) } #[inline] @@ -220,26 +227,27 @@ impl ScalarInt { // `into` performed sign extension, we have to truncate let truncated = size.truncate(i as u128); if size.sign_extend(truncated) as i128 == i { - Some(Self { data: truncated, size: NonZero::new(size.bytes() as u8).unwrap() }) + Some(Self::raw(truncated, size)) } else { None } } + /// Returns the truncated result, and whether truncation changed the value. #[inline] - pub fn try_from_target_usize(i: impl Into<u128>, tcx: TyCtxt<'_>) -> Option<Self> { - Self::try_from_uint(i, tcx.data_layout.pointer_size) + pub fn truncate_from_int(i: impl Into<i128>, size: Size) -> (Self, bool) { + let data = i.into(); + let r = Self::raw(size.truncate(data as u128), size); + (r, size.sign_extend(r.data) as i128 != data) } #[inline] - pub fn assert_bits(self, target_size: Size) -> u128 { - self.to_bits(target_size).unwrap_or_else(|size| { - bug!("expected int of size {}, but got size {}", target_size.bytes(), size.bytes()) - }) + pub fn try_from_target_usize(i: impl Into<u128>, tcx: TyCtxt<'_>) -> Option<Self> { + Self::try_from_uint(i, tcx.data_layout.pointer_size) } #[inline] - pub fn to_bits(self, target_size: Size) -> Result<u128, Size> { + pub fn try_to_bits(self, target_size: Size) -> Result<u128, Size> { assert_ne!(target_size.bytes(), 0, "you should never look at the bits of a ZST"); if target_size.bytes() == u64::from(self.size.get()) { self.check_data(); @@ -249,16 +257,28 @@ impl ScalarInt { } } + #[inline] + pub fn assert_bits(self, target_size: Size) -> u128 { + self.try_to_bits(target_size).unwrap_or_else(|size| { + bug!("expected int of size {}, but got size {}", target_size.bytes(), size.bytes()) + }) + } + /// Tries to convert the `ScalarInt` to an unsigned integer of the given size. /// Fails if the size of the `ScalarInt` is not equal to `size` and returns the /// `ScalarInt`s size in that case. #[inline] pub fn try_to_uint(self, size: Size) -> Result<u128, Size> { - self.to_bits(size) + self.try_to_bits(size) + } + + #[inline] + pub fn assert_uint(self, size: Size) -> u128 { + self.assert_bits(size) } // Tries to convert the `ScalarInt` to `u8`. Fails if the `size` of the `ScalarInt` - // in not equal to `Size { raw: 1 }` and returns the `size` value of the `ScalarInt` in + // in not equal to 1 byte and returns the `size` value of the `ScalarInt` in // that case. #[inline] pub fn try_to_u8(self) -> Result<u8, Size> { @@ -266,7 +286,7 @@ impl ScalarInt { } /// Tries to convert the `ScalarInt` to `u16`. Fails if the size of the `ScalarInt` - /// in not equal to `Size { raw: 2 }` and returns the `size` value of the `ScalarInt` in + /// in not equal to 2 bytes and returns the `size` value of the `ScalarInt` in /// that case. #[inline] pub fn try_to_u16(self) -> Result<u16, Size> { @@ -274,7 +294,7 @@ impl ScalarInt { } /// Tries to convert the `ScalarInt` to `u32`. Fails if the `size` of the `ScalarInt` - /// in not equal to `Size { raw: 4 }` and returns the `size` value of the `ScalarInt` in + /// in not equal to 4 bytes and returns the `size` value of the `ScalarInt` in /// that case. #[inline] pub fn try_to_u32(self) -> Result<u32, Size> { @@ -282,7 +302,7 @@ impl ScalarInt { } /// Tries to convert the `ScalarInt` to `u64`. Fails if the `size` of the `ScalarInt` - /// in not equal to `Size { raw: 8 }` and returns the `size` value of the `ScalarInt` in + /// in not equal to 8 bytes and returns the `size` value of the `ScalarInt` in /// that case. #[inline] pub fn try_to_u64(self) -> Result<u64, Size> { @@ -290,7 +310,7 @@ impl ScalarInt { } /// Tries to convert the `ScalarInt` to `u128`. Fails if the `size` of the `ScalarInt` - /// in not equal to `Size { raw: 16 }` and returns the `size` value of the `ScalarInt` in + /// in not equal to 16 bytes and returns the `size` value of the `ScalarInt` in /// that case. #[inline] pub fn try_to_u128(self) -> Result<u128, Size> { @@ -303,7 +323,7 @@ impl ScalarInt { } // Tries to convert the `ScalarInt` to `bool`. Fails if the `size` of the `ScalarInt` - // in not equal to `Size { raw: 1 }` or if the value is not 0 or 1 and returns the `size` + // in not equal to 1 byte or if the value is not 0 or 1 and returns the `size` // value of the `ScalarInt` in that case. #[inline] pub fn try_to_bool(self) -> Result<bool, Size> { @@ -319,40 +339,46 @@ impl ScalarInt { /// `ScalarInt`s size in that case. #[inline] pub fn try_to_int(self, size: Size) -> Result<i128, Size> { - let b = self.to_bits(size)?; + let b = self.try_to_bits(size)?; Ok(size.sign_extend(b) as i128) } + #[inline] + pub fn assert_int(self, size: Size) -> i128 { + let b = self.assert_bits(size); + size.sign_extend(b) as i128 + } + /// Tries to convert the `ScalarInt` to i8. - /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 1 }` + /// Fails if the size of the `ScalarInt` is not equal to 1 byte /// and returns the `ScalarInt`s size in that case. pub fn try_to_i8(self) -> Result<i8, Size> { self.try_to_int(Size::from_bits(8)).map(|v| i8::try_from(v).unwrap()) } /// Tries to convert the `ScalarInt` to i16. - /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 2 }` + /// Fails if the size of the `ScalarInt` is not equal to 2 bytes /// and returns the `ScalarInt`s size in that case. pub fn try_to_i16(self) -> Result<i16, Size> { self.try_to_int(Size::from_bits(16)).map(|v| i16::try_from(v).unwrap()) } /// Tries to convert the `ScalarInt` to i32. - /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 4 }` + /// Fails if the size of the `ScalarInt` is not equal to 4 bytes /// and returns the `ScalarInt`s size in that case. pub fn try_to_i32(self) -> Result<i32, Size> { self.try_to_int(Size::from_bits(32)).map(|v| i32::try_from(v).unwrap()) } /// Tries to convert the `ScalarInt` to i64. - /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 8 }` + /// Fails if the size of the `ScalarInt` is not equal to 8 bytes /// and returns the `ScalarInt`s size in that case. pub fn try_to_i64(self) -> Result<i64, Size> { self.try_to_int(Size::from_bits(64)).map(|v| i64::try_from(v).unwrap()) } /// Tries to convert the `ScalarInt` to i128. - /// Fails if the size of the `ScalarInt` is not equal to `Size { raw: 16 }` + /// Fails if the size of the `ScalarInt` is not equal to 16 bytes /// and returns the `ScalarInt`s size in that case. pub fn try_to_i128(self) -> Result<i128, Size> { self.try_to_int(Size::from_bits(128)) @@ -366,7 +392,7 @@ impl ScalarInt { #[inline] pub fn try_to_float<F: Float>(self) -> Result<F, Size> { // Going through `to_uint` to check size and truncation. - Ok(F::from_bits(self.to_bits(Size::from_bits(F::BITS))?)) + Ok(F::from_bits(self.try_to_bits(Size::from_bits(F::BITS))?)) } #[inline] @@ -415,7 +441,7 @@ macro_rules! try_from { fn try_from(int: ScalarInt) -> Result<Self, Size> { // The `unwrap` cannot fail because to_bits (if it succeeds) // is guaranteed to return a value that fits into the size. - int.to_bits(Size::from_bytes(std::mem::size_of::<$ty>())) + int.try_to_bits(Size::from_bytes(std::mem::size_of::<$ty>())) .map(|u| u.try_into().unwrap()) } } @@ -450,7 +476,7 @@ impl TryFrom<ScalarInt> for char { #[inline] fn try_from(int: ScalarInt) -> Result<Self, Self::Error> { - let Ok(bits) = int.to_bits(Size::from_bytes(std::mem::size_of::<char>())) else { + let Ok(bits) = int.try_to_bits(Size::from_bytes(std::mem::size_of::<char>())) else { return Err(CharTryFromScalarInt); }; match char::from_u32(bits.try_into().unwrap()) { @@ -472,7 +498,7 @@ impl TryFrom<ScalarInt> for Half { type Error = Size; #[inline] fn try_from(int: ScalarInt) -> Result<Self, Size> { - int.to_bits(Size::from_bytes(2)).map(Self::from_bits) + int.try_to_bits(Size::from_bytes(2)).map(Self::from_bits) } } @@ -488,7 +514,7 @@ impl TryFrom<ScalarInt> for Single { type Error = Size; #[inline] fn try_from(int: ScalarInt) -> Result<Self, Size> { - int.to_bits(Size::from_bytes(4)).map(Self::from_bits) + int.try_to_bits(Size::from_bytes(4)).map(Self::from_bits) } } @@ -504,7 +530,7 @@ impl TryFrom<ScalarInt> for Double { type Error = Size; #[inline] fn try_from(int: ScalarInt) -> Result<Self, Size> { - int.to_bits(Size::from_bytes(8)).map(Self::from_bits) + int.try_to_bits(Size::from_bytes(8)).map(Self::from_bits) } } @@ -520,7 +546,7 @@ impl TryFrom<ScalarInt> for Quad { type Error = Size; #[inline] fn try_from(int: ScalarInt) -> Result<Self, Size> { - int.to_bits(Size::from_bytes(16)).map(Self::from_bits) + int.try_to_bits(Size::from_bytes(16)).map(Self::from_bits) } } diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs index 94e41709f5d..a7e0a0402ce 100644 --- a/compiler/rustc_middle/src/ty/consts/kind.rs +++ b/compiler/rustc_middle/src/ty/consts/kind.rs @@ -71,8 +71,5 @@ pub enum Expr<'tcx> { Cast(CastKind, Const<'tcx>, Ty<'tcx>), } -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(Expr<'_>, 24); - -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] -static_assert_size!(super::ConstKind<'_>, 32); diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs index ffa0e89c473..96bc5515a56 100644 --- a/compiler/rustc_middle/src/ty/consts/valtree.rs +++ b/compiler/rustc_middle/src/ty/consts/valtree.rs @@ -3,7 +3,7 @@ use crate::mir::interpret::Scalar; use crate::ty::{self, Ty, TyCtxt}; use rustc_macros::{HashStable, TyDecodable, TyEncodable}; -#[derive(Copy, Clone, Debug, Hash, TyEncodable, TyDecodable, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Copy, Clone, Debug, Hash, TyEncodable, TyDecodable, Eq, PartialEq)] #[derive(HashStable)] /// This datastructure is used to represent the value of constants used in the type system. /// diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 50e68bfdbe7..6381bd190ac 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -15,7 +15,9 @@ use rustc_span::symbol::{sym, Symbol}; use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP}; use rustc_target::abi::call::FnAbi; use rustc_target::abi::*; -use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target}; +use rustc_target::spec::{ + abi::Abi as SpecAbi, HasTargetSpec, HasWasmCAbiOpt, PanicStrategy, Target, WasmCAbi, +}; use std::borrow::Cow; use std::cmp; @@ -483,6 +485,12 @@ impl<'tcx> HasTargetSpec for TyCtxt<'tcx> { } } +impl<'tcx> HasWasmCAbiOpt for TyCtxt<'tcx> { + fn wasm_c_abi_opt(&self) -> WasmCAbi { + self.sess.opts.unstable_opts.wasm_c_abi + } +} + impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> { #[inline] fn tcx(&self) -> TyCtxt<'tcx> { @@ -528,6 +536,12 @@ impl<'tcx, T: HasTargetSpec> HasTargetSpec for LayoutCx<'tcx, T> { } } +impl<'tcx, T: HasWasmCAbiOpt> HasWasmCAbiOpt for LayoutCx<'tcx, T> { + fn wasm_c_abi_opt(&self) -> WasmCAbi { + self.tcx.wasm_c_abi_opt() + } +} + impl<'tcx, T: HasTyCtxt<'tcx>> HasTyCtxt<'tcx> for LayoutCx<'tcx, T> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx.tcx() diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index e6b773ae512..d0a69c6fd2c 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -2183,7 +2183,7 @@ pub struct DestructuredConst<'tcx> { } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 90c68e7ddfc..14a77d4b37e 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -427,6 +427,7 @@ TrivialTypeTraversalImpls! { crate::mir::coverage::BlockMarkerId, crate::mir::coverage::CounterId, crate::mir::coverage::ExpressionId, + crate::mir::coverage::ConditionId, crate::mir::Local, crate::mir::Promoted, crate::traits::Reveal, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index ad64745d579..6084e8d7cab 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2697,7 +2697,7 @@ impl<'tcx> VarianceDiagInfo<'tcx> { } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl index 1de691f32a7..34440c60cf3 100644 --- a/compiler/rustc_mir_build/messages.ftl +++ b/compiler/rustc_mir_build/messages.ftl @@ -97,6 +97,8 @@ mir_build_deref_raw_pointer_requires_unsafe_unsafe_op_in_unsafe_fn_allowed = .note = raw pointers may be null, dangling or unaligned; they can violate aliasing rules and cause data races: all of these are undefined behavior .label = dereference of raw pointer +mir_build_exceeds_mcdc_condition_num_limit = Conditions number of the decision ({$conditions_num}) exceeds limit ({$max_conditions_num}). MCDC analysis will not count this expression. + mir_build_extern_static_requires_unsafe = use of extern static is unsafe and requires unsafe block .note = extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior diff --git a/compiler/rustc_mir_build/src/build/coverageinfo.rs b/compiler/rustc_mir_build/src/build/coverageinfo.rs index ab0043906b1..57f809ef7cf 100644 --- a/compiler/rustc_mir_build/src/build/coverageinfo.rs +++ b/compiler/rustc_mir_build/src/build/coverageinfo.rs @@ -1,14 +1,20 @@ use std::assert_matches::assert_matches; use std::collections::hash_map::Entry; +use std::collections::VecDeque; use rustc_data_structures::fx::FxHashMap; -use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageKind}; +use rustc_middle::mir::coverage::{ + BlockMarkerId, BranchSpan, ConditionId, ConditionInfo, CoverageKind, MCDCBranchSpan, + MCDCDecisionSpan, +}; use rustc_middle::mir::{self, BasicBlock, UnOp}; -use rustc_middle::thir::{ExprId, ExprKind, Thir}; +use rustc_middle::thir::{ExprId, ExprKind, LogicalOp, Thir}; use rustc_middle::ty::TyCtxt; use rustc_span::def_id::LocalDefId; +use rustc_span::Span; use crate::build::Builder; +use crate::errors::MCDCExceedsConditionNumLimit; pub(crate) struct BranchInfoBuilder { /// Maps condition expressions to their enclosing `!`, for better instrumentation. @@ -16,6 +22,9 @@ pub(crate) struct BranchInfoBuilder { num_block_markers: usize, branch_spans: Vec<BranchSpan>, + mcdc_branch_spans: Vec<MCDCBranchSpan>, + mcdc_decision_spans: Vec<MCDCDecisionSpan>, + mcdc_state: Option<MCDCState>, } #[derive(Clone, Copy)] @@ -33,7 +42,14 @@ impl BranchInfoBuilder { /// is enabled and `def_id` represents a function that is eligible for coverage. pub(crate) fn new_if_enabled(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<Self> { if tcx.sess.instrument_coverage_branch() && tcx.is_eligible_for_coverage(def_id) { - Some(Self { nots: FxHashMap::default(), num_block_markers: 0, branch_spans: vec![] }) + Some(Self { + nots: FxHashMap::default(), + num_block_markers: 0, + branch_spans: vec![], + mcdc_branch_spans: vec![], + mcdc_decision_spans: vec![], + mcdc_state: MCDCState::new_if_enabled(tcx), + }) } else { None } @@ -79,6 +95,55 @@ impl BranchInfoBuilder { } } + fn record_conditions_operation(&mut self, logical_op: LogicalOp, span: Span) { + if let Some(mcdc_state) = self.mcdc_state.as_mut() { + mcdc_state.record_conditions(logical_op, span); + } + } + + fn fetch_condition_info( + &mut self, + tcx: TyCtxt<'_>, + true_marker: BlockMarkerId, + false_marker: BlockMarkerId, + ) -> Option<ConditionInfo> { + let mcdc_state = self.mcdc_state.as_mut()?; + let (mut condition_info, decision_result) = + mcdc_state.take_condition(true_marker, false_marker); + if let Some(decision) = decision_result { + match decision.conditions_num { + 0 => { + unreachable!("Decision with no condition is not expected"); + } + 1..=MAX_CONDITIONS_NUM_IN_DECISION => { + self.mcdc_decision_spans.push(decision); + } + _ => { + // Do not generate mcdc mappings and statements for decisions with too many conditions. + let rebase_idx = self.mcdc_branch_spans.len() - decision.conditions_num + 1; + let to_normal_branches = self.mcdc_branch_spans.split_off(rebase_idx); + self.branch_spans.extend(to_normal_branches.into_iter().map( + |MCDCBranchSpan { span, true_marker, false_marker, .. }| BranchSpan { + span, + true_marker, + false_marker, + }, + )); + + // ConditionInfo of this branch shall also be reset. + condition_info = None; + + tcx.dcx().emit_warn(MCDCExceedsConditionNumLimit { + span: decision.span, + conditions_num: decision.conditions_num, + max_conditions_num: MAX_CONDITIONS_NUM_IN_DECISION, + }); + } + } + } + condition_info + } + fn next_block_marker_id(&mut self) -> BlockMarkerId { let id = BlockMarkerId::from_usize(self.num_block_markers); self.num_block_markers += 1; @@ -86,14 +151,167 @@ impl BranchInfoBuilder { } pub(crate) fn into_done(self) -> Option<Box<mir::coverage::BranchInfo>> { - let Self { nots: _, num_block_markers, branch_spans } = self; + let Self { + nots: _, + num_block_markers, + branch_spans, + mcdc_branch_spans, + mcdc_decision_spans, + .. + } = self; if num_block_markers == 0 { assert!(branch_spans.is_empty()); return None; } - Some(Box::new(mir::coverage::BranchInfo { num_block_markers, branch_spans })) + Some(Box::new(mir::coverage::BranchInfo { + num_block_markers, + branch_spans, + mcdc_branch_spans, + mcdc_decision_spans, + })) + } +} + +/// The MCDC bitmap scales exponentially (2^n) based on the number of conditions seen, +/// So llvm sets a maximum value prevents the bitmap footprint from growing too large without the user's knowledge. +/// This limit may be relaxed if the [upstream change](https://github.com/llvm/llvm-project/pull/82448) is merged. +const MAX_CONDITIONS_NUM_IN_DECISION: usize = 6; + +struct MCDCState { + /// To construct condition evaluation tree. + decision_stack: VecDeque<ConditionInfo>, + processing_decision: Option<MCDCDecisionSpan>, +} + +impl MCDCState { + fn new_if_enabled(tcx: TyCtxt<'_>) -> Option<Self> { + tcx.sess + .instrument_coverage_mcdc() + .then(|| Self { decision_stack: VecDeque::new(), processing_decision: None }) + } + + // At first we assign ConditionIds for each sub expression. + // If the sub expression is composite, re-assign its ConditionId to its LHS and generate a new ConditionId for its RHS. + // + // Example: "x = (A && B) || (C && D) || (D && F)" + // + // Visit Depth1: + // (A && B) || (C && D) || (D && F) + // ^-------LHS--------^ ^-RHS--^ + // ID=1 ID=2 + // + // Visit LHS-Depth2: + // (A && B) || (C && D) + // ^-LHS--^ ^-RHS--^ + // ID=1 ID=3 + // + // Visit LHS-Depth3: + // (A && B) + // LHS RHS + // ID=1 ID=4 + // + // Visit RHS-Depth3: + // (C && D) + // LHS RHS + // ID=3 ID=5 + // + // Visit RHS-Depth2: (D && F) + // LHS RHS + // ID=2 ID=6 + // + // Visit Depth1: + // (A && B) || (C && D) || (D && F) + // ID=1 ID=4 ID=3 ID=5 ID=2 ID=6 + // + // A node ID of '0' always means MC/DC isn't being tracked. + // + // If a "next" node ID is '0', it means it's the end of the test vector. + // + // As the compiler tracks expression in pre-order, we can ensure that condition info of parents are always properly assigned when their children are visited. + // - If the op is AND, the "false_next" of LHS and RHS should be the parent's "false_next". While "true_next" of the LHS is the RHS, the "true next" of RHS is the parent's "true_next". + // - If the op is OR, the "true_next" of LHS and RHS should be the parent's "true_next". While "false_next" of the LHS is the RHS, the "false next" of RHS is the parent's "false_next". + fn record_conditions(&mut self, op: LogicalOp, span: Span) { + let decision = match self.processing_decision.as_mut() { + Some(decision) => { + decision.span = decision.span.to(span); + decision + } + None => self.processing_decision.insert(MCDCDecisionSpan { + span, + conditions_num: 0, + end_markers: vec![], + }), + }; + + let parent_condition = self.decision_stack.pop_back().unwrap_or_default(); + let lhs_id = if parent_condition.condition_id == ConditionId::NONE { + decision.conditions_num += 1; + ConditionId::from(decision.conditions_num) + } else { + parent_condition.condition_id + }; + + decision.conditions_num += 1; + let rhs_condition_id = ConditionId::from(decision.conditions_num); + + let (lhs, rhs) = match op { + LogicalOp::And => { + let lhs = ConditionInfo { + condition_id: lhs_id, + true_next_id: rhs_condition_id, + false_next_id: parent_condition.false_next_id, + }; + let rhs = ConditionInfo { + condition_id: rhs_condition_id, + true_next_id: parent_condition.true_next_id, + false_next_id: parent_condition.false_next_id, + }; + (lhs, rhs) + } + LogicalOp::Or => { + let lhs = ConditionInfo { + condition_id: lhs_id, + true_next_id: parent_condition.true_next_id, + false_next_id: rhs_condition_id, + }; + let rhs = ConditionInfo { + condition_id: rhs_condition_id, + true_next_id: parent_condition.true_next_id, + false_next_id: parent_condition.false_next_id, + }; + (lhs, rhs) + } + }; + // We visit expressions tree in pre-order, so place the left-hand side on the top. + self.decision_stack.push_back(rhs); + self.decision_stack.push_back(lhs); + } + + fn take_condition( + &mut self, + true_marker: BlockMarkerId, + false_marker: BlockMarkerId, + ) -> (Option<ConditionInfo>, Option<MCDCDecisionSpan>) { + let Some(condition_info) = self.decision_stack.pop_back() else { + return (None, None); + }; + let Some(decision) = self.processing_decision.as_mut() else { + bug!("Processing decision should have been created before any conditions are taken"); + }; + if condition_info.true_next_id == ConditionId::NONE { + decision.end_markers.push(true_marker); + } + if condition_info.false_next_id == ConditionId::NONE { + decision.end_markers.push(false_marker); + } + + if self.decision_stack.is_empty() { + (Some(condition_info), self.processing_decision.take()) + } else { + (Some(condition_info), None) + } } } @@ -137,10 +355,27 @@ impl Builder<'_, '_> { let true_marker = inject_branch_marker(then_block); let false_marker = inject_branch_marker(else_block); - branch_info.branch_spans.push(BranchSpan { - span: source_info.span, - true_marker, - false_marker, - }); + if let Some(condition_info) = + branch_info.fetch_condition_info(self.tcx, true_marker, false_marker) + { + branch_info.mcdc_branch_spans.push(MCDCBranchSpan { + span: source_info.span, + condition_info, + true_marker, + false_marker, + }); + } else { + branch_info.branch_spans.push(BranchSpan { + span: source_info.span, + true_marker, + false_marker, + }); + } + } + + pub(crate) fn visit_coverage_branch_operation(&mut self, logical_op: LogicalOp, span: Span) { + if let Some(branch_info) = self.coverage_branch_info.as_mut() { + branch_info.record_conditions_operation(logical_op, span); + } } } diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs index 9730473c428..f46dceeeedf 100644 --- a/compiler/rustc_mir_build/src/build/matches/mod.rs +++ b/compiler/rustc_mir_build/src/build/matches/mod.rs @@ -77,11 +77,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { match expr.kind { ExprKind::LogicalOp { op: LogicalOp::And, lhs, rhs } => { + this.visit_coverage_branch_operation(LogicalOp::And, expr_span); let lhs_then_block = unpack!(this.then_else_break_inner(block, lhs, args)); let rhs_then_block = unpack!(this.then_else_break_inner(lhs_then_block, rhs, args)); rhs_then_block.unit() } ExprKind::LogicalOp { op: LogicalOp::Or, lhs, rhs } => { + this.visit_coverage_branch_operation(LogicalOp::Or, expr_span); let local_scope = this.local_scope(); let (lhs_success_block, failure_block) = this.in_if_then_scope(local_scope, expr_span, |this| { diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index cc525f2ac2f..3c18afe1a78 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -1023,7 +1023,13 @@ pub(crate) fn parse_float_into_scalar( let num = num.as_str(); match float_ty { // FIXME(f16_f128): When available, compare to the library parser as with `f32` and `f64` - ty::FloatTy::F16 => num.parse::<Half>().ok().map(Scalar::from_f16), + ty::FloatTy::F16 => { + let mut f = num.parse::<Half>().ok()?; + if neg { + f = -f; + } + Some(Scalar::from_f16(f)) + } ty::FloatTy::F32 => { let Ok(rust_f) = num.parse::<f32>() else { return None }; let mut f = num @@ -1071,7 +1077,13 @@ pub(crate) fn parse_float_into_scalar( Some(Scalar::from_f64(f)) } // FIXME(f16_f128): When available, compare to the library parser as with `f32` and `f64` - ty::FloatTy::F128 => num.parse::<Quad>().ok().map(Scalar::from_f128), + ty::FloatTy::F128 => { + let mut f = num.parse::<Quad>().ok()?; + if neg { + f = -f; + } + Some(Scalar::from_f128(f)) + } } } diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs index 26f10fdd333..9ddfb12bf76 100644 --- a/compiler/rustc_mir_build/src/errors.rs +++ b/compiler/rustc_mir_build/src/errors.rs @@ -819,6 +819,15 @@ pub struct NontrivialStructuralMatch<'tcx> { } #[derive(Diagnostic)] +#[diag(mir_build_exceeds_mcdc_condition_num_limit)] +pub(crate) struct MCDCExceedsConditionNumLimit { + #[primary_span] + pub span: Span, + pub conditions_num: usize, + pub max_conditions_num: usize, +} + +#[derive(Diagnostic)] #[diag(mir_build_pattern_not_covered, code = E0005)] pub(crate) struct PatternNotCovered<'s, 'tcx> { #[primary_span] diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index d382d2c03c2..9e8648b0f93 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -100,9 +100,12 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: &coverage_counters, ); + inject_mcdc_statements(mir_body, &basic_coverage_blocks, &coverage_spans); + mir_body.function_coverage_info = Some(Box::new(FunctionCoverageInfo { function_source_hash: hir_info.function_source_hash, num_counters: coverage_counters.num_counters(), + mcdc_bitmap_bytes: coverage_spans.test_vector_bitmap_bytes(), expressions: coverage_counters.into_expressions(), mappings, })); @@ -136,20 +139,33 @@ fn create_mappings<'tcx>( .as_term() }; - coverage_spans - .all_bcb_mappings() - .filter_map(|&BcbMapping { kind: bcb_mapping_kind, span }| { - let kind = match bcb_mapping_kind { + let mut mappings = Vec::new(); + + mappings.extend(coverage_spans.all_bcb_mappings().filter_map( + |BcbMapping { kind: bcb_mapping_kind, span }| { + let kind = match *bcb_mapping_kind { BcbMappingKind::Code(bcb) => MappingKind::Code(term_for_bcb(bcb)), BcbMappingKind::Branch { true_bcb, false_bcb } => MappingKind::Branch { true_term: term_for_bcb(true_bcb), false_term: term_for_bcb(false_bcb), }, + BcbMappingKind::MCDCBranch { true_bcb, false_bcb, condition_info } => { + MappingKind::MCDCBranch { + true_term: term_for_bcb(true_bcb), + false_term: term_for_bcb(false_bcb), + mcdc_params: condition_info, + } + } + BcbMappingKind::MCDCDecision { bitmap_idx, conditions_num, .. } => { + MappingKind::MCDCDecision(DecisionInfo { bitmap_idx, conditions_num }) + } }; - let code_region = make_code_region(source_map, file_name, span, body_span)?; + let code_region = make_code_region(source_map, file_name, *span, body_span)?; Some(Mapping { kind, code_region }) - }) - .collect::<Vec<_>>() + }, + )); + + mappings } /// For each BCB node or BCB edge that has an associated coverage counter, @@ -204,6 +220,55 @@ fn inject_coverage_statements<'tcx>( } } +/// For each conditions inject statements to update condition bitmap after it has been evaluated. +/// For each decision inject statements to update test vector bitmap after it has been evaluated. +fn inject_mcdc_statements<'tcx>( + mir_body: &mut mir::Body<'tcx>, + basic_coverage_blocks: &CoverageGraph, + coverage_spans: &CoverageSpans, +) { + if coverage_spans.test_vector_bitmap_bytes() == 0 { + return; + } + + // Inject test vector update first because `inject_statement` always insert new statement at head. + for (end_bcbs, bitmap_idx) in + coverage_spans.all_bcb_mappings().filter_map(|mapping| match &mapping.kind { + BcbMappingKind::MCDCDecision { end_bcbs, bitmap_idx, .. } => { + Some((end_bcbs, *bitmap_idx)) + } + _ => None, + }) + { + for end in end_bcbs { + let end_bb = basic_coverage_blocks[*end].leader_bb(); + inject_statement(mir_body, CoverageKind::TestVectorBitmapUpdate { bitmap_idx }, end_bb); + } + } + + for (true_bcb, false_bcb, condition_id) in + coverage_spans.all_bcb_mappings().filter_map(|mapping| match mapping.kind { + BcbMappingKind::MCDCBranch { true_bcb, false_bcb, condition_info } => { + Some((true_bcb, false_bcb, condition_info.condition_id)) + } + _ => None, + }) + { + let true_bb = basic_coverage_blocks[true_bcb].leader_bb(); + inject_statement( + mir_body, + CoverageKind::CondBitmapUpdate { id: condition_id, value: true }, + true_bb, + ); + let false_bb = basic_coverage_blocks[false_bcb].leader_bb(); + inject_statement( + mir_body, + CoverageKind::CondBitmapUpdate { id: condition_id, value: false }, + false_bb, + ); + } +} + /// Given two basic blocks that have a control-flow edge between them, creates /// and returns a new block that sits between those blocks. fn inject_edge_counter_basic_block( diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 65715253647..f77ee63d02c 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -61,7 +61,17 @@ fn coverage_ids_info<'tcx>( .max() .unwrap_or(CounterId::ZERO); - CoverageIdsInfo { max_counter_id } + let mcdc_bitmap_bytes = mir_body + .coverage_branch_info + .as_deref() + .map(|info| { + info.mcdc_decision_spans + .iter() + .fold(0, |acc, decision| acc + (1_u32 << decision.conditions_num).div_ceil(8)) + }) + .unwrap_or_default(); + + CoverageIdsInfo { max_counter_id, mcdc_bitmap_bytes } } fn all_coverage_in_mir_body<'a, 'tcx>( diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 03ede886688..a4cd8a38c66 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,7 +1,9 @@ use rustc_data_structures::graph::DirectedGraph; use rustc_index::bit_set::BitSet; use rustc_middle::mir; +use rustc_middle::mir::coverage::ConditionInfo; use rustc_span::{BytePos, Span}; +use std::collections::BTreeSet; use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, START_BCB}; use crate::coverage::spans::from_mir::SpanFromMir; @@ -9,12 +11,20 @@ use crate::coverage::ExtractedHirInfo; mod from_mir; -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] pub(super) enum BcbMappingKind { /// Associates an ordinary executable code span with its corresponding BCB. Code(BasicCoverageBlock), /// Associates a branch span with BCBs for its true and false arms. Branch { true_bcb: BasicCoverageBlock, false_bcb: BasicCoverageBlock }, + /// Associates a mcdc branch span with condition info besides fields for normal branch. + MCDCBranch { + true_bcb: BasicCoverageBlock, + false_bcb: BasicCoverageBlock, + condition_info: ConditionInfo, + }, + /// Associates a mcdc decision with its join BCB. + MCDCDecision { end_bcbs: BTreeSet<BasicCoverageBlock>, bitmap_idx: u32, conditions_num: u16 }, } #[derive(Debug)] @@ -26,6 +36,7 @@ pub(super) struct BcbMapping { pub(super) struct CoverageSpans { bcb_has_mappings: BitSet<BasicCoverageBlock>, mappings: Vec<BcbMapping>, + test_vector_bitmap_bytes: u32, } impl CoverageSpans { @@ -36,6 +47,10 @@ impl CoverageSpans { pub(super) fn all_bcb_mappings(&self) -> impl Iterator<Item = &BcbMapping> { self.mappings.iter() } + + pub(super) fn test_vector_bitmap_bytes(&self) -> u32 { + self.test_vector_bitmap_bytes + } } /// Extracts coverage-relevant spans from MIR, and associates them with @@ -85,17 +100,26 @@ pub(super) fn generate_coverage_spans( let mut insert = |bcb| { bcb_has_mappings.insert(bcb); }; - for &BcbMapping { kind, span: _ } in &mappings { - match kind { + let mut test_vector_bitmap_bytes = 0; + for BcbMapping { kind, span: _ } in &mappings { + match *kind { BcbMappingKind::Code(bcb) => insert(bcb), - BcbMappingKind::Branch { true_bcb, false_bcb } => { + BcbMappingKind::Branch { true_bcb, false_bcb } + | BcbMappingKind::MCDCBranch { true_bcb, false_bcb, .. } => { insert(true_bcb); insert(false_bcb); } + BcbMappingKind::MCDCDecision { bitmap_idx, conditions_num, .. } => { + // `bcb_has_mappings` is used for inject coverage counters + // but they are not needed for decision BCBs. + // While the length of test vector bitmap should be calculated here. + test_vector_bitmap_bytes = test_vector_bitmap_bytes + .max(bitmap_idx + (1_u32 << conditions_num as u32).div_ceil(8)); + } } } - Some(CoverageSpans { bcb_has_mappings, mappings }) + Some(CoverageSpans { bcb_has_mappings, mappings, test_vector_bitmap_bytes }) } #[derive(Debug)] diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index adb0c9f1929..b9919a2ae88 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -1,7 +1,9 @@ use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexVec; -use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageKind}; +use rustc_middle::mir::coverage::{ + BlockMarkerId, BranchSpan, CoverageKind, MCDCBranchSpan, MCDCDecisionSpan, +}; use rustc_middle::mir::{ self, AggregateKind, BasicBlock, FakeReadCause, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, @@ -227,7 +229,10 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { // These coverage statements should not exist prior to coverage instrumentation. StatementKind::Coverage( - CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. }, + CoverageKind::CounterIncrement { .. } + | CoverageKind::ExpressionUsed { .. } + | CoverageKind::CondBitmapUpdate { .. } + | CoverageKind::TestVectorBitmapUpdate { .. }, ) => bug!( "Unexpected coverage statement found during coverage instrumentation: {statement:?}" ), @@ -384,10 +389,11 @@ pub(super) fn extract_branch_mappings( } } - branch_info - .branch_spans - .iter() - .filter_map(|&BranchSpan { span: raw_span, true_marker, false_marker }| { + let bcb_from_marker = + |marker: BlockMarkerId| basic_coverage_blocks.bcb_from_bb(block_markers[marker]?); + + let check_branch_bcb = + |raw_span: Span, true_marker: BlockMarkerId, false_marker: BlockMarkerId| { // For now, ignore any branch span that was introduced by // expansion. This makes things like assert macros less noisy. if !raw_span.ctxt().outer_expn_data().is_root() { @@ -395,13 +401,56 @@ pub(super) fn extract_branch_mappings( } let (span, _) = unexpand_into_body_span_with_visible_macro(raw_span, body_span)?; - let bcb_from_marker = - |marker: BlockMarkerId| basic_coverage_blocks.bcb_from_bb(block_markers[marker]?); - let true_bcb = bcb_from_marker(true_marker)?; let false_bcb = bcb_from_marker(false_marker)?; + Some((span, true_bcb, false_bcb)) + }; - Some(BcbMapping { kind: BcbMappingKind::Branch { true_bcb, false_bcb }, span }) + let branch_filter_map = |&BranchSpan { span: raw_span, true_marker, false_marker }| { + check_branch_bcb(raw_span, true_marker, false_marker).map(|(span, true_bcb, false_bcb)| { + BcbMapping { kind: BcbMappingKind::Branch { true_bcb, false_bcb }, span } }) + }; + + let mcdc_branch_filter_map = + |&MCDCBranchSpan { span: raw_span, true_marker, false_marker, condition_info }| { + check_branch_bcb(raw_span, true_marker, false_marker).map( + |(span, true_bcb, false_bcb)| BcbMapping { + kind: BcbMappingKind::MCDCBranch { true_bcb, false_bcb, condition_info }, + span, + }, + ) + }; + + let mut next_bitmap_idx = 0; + + let decision_filter_map = |decision: &MCDCDecisionSpan| { + let (span, _) = unexpand_into_body_span_with_visible_macro(decision.span, body_span)?; + + let end_bcbs = decision + .end_markers + .iter() + .map(|&marker| bcb_from_marker(marker)) + .collect::<Option<_>>()?; + + let bitmap_idx = next_bitmap_idx; + next_bitmap_idx += (1_u32 << decision.conditions_num).div_ceil(8); + + Some(BcbMapping { + kind: BcbMappingKind::MCDCDecision { + end_bcbs, + bitmap_idx, + conditions_num: decision.conditions_num as u16, + }, + span, + }) + }; + + branch_info + .branch_spans + .iter() + .filter_map(branch_filter_map) + .chain(branch_info.mcdc_branch_spans.iter().filter_map(mcdc_branch_filter_map)) + .chain(branch_info.mcdc_decision_spans.iter().filter_map(decision_filter_map)) .collect::<Vec<_>>() } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 60513a674af..625d8f53939 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -11,7 +11,7 @@ use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; -use rustc_session::config::OptLevel; +use rustc_session::config::{DebugInfo, OptLevel}; use rustc_span::source_map::Spanned; use rustc_span::sym; use rustc_target::abi::FieldIdx; @@ -699,7 +699,19 @@ impl<'tcx> Inliner<'tcx> { // Insert all of the (mapped) parts of the callee body into the caller. caller_body.local_decls.extend(callee_body.drain_vars_and_temps()); caller_body.source_scopes.extend(&mut callee_body.source_scopes.drain(..)); - caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + if self + .tcx + .sess + .opts + .unstable_opts + .inline_mir_preserve_debug + .unwrap_or(self.tcx.sess.opts.debuginfo != DebugInfo::None) + { + // Note that we need to preserve these in the standard library so that + // people working on rust can build with or without debuginfo while + // still getting consistent results from the mir-opt tests. + caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + } caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..)); caller_body[callsite.block].terminator = Some(Terminator { diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 2218154ea5e..2744026a7c9 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -796,7 +796,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { if let Some(ref value) = self.eval_operand(discr) && let Some(value_const) = self.use_ecx(|this| this.ecx.read_scalar(value)) && let Ok(constant) = value_const.try_to_int() - && let Ok(constant) = constant.to_bits(constant.size()) + && let Ok(constant) = constant.try_to_bits(constant.size()) { // We managed to evaluate the discriminant, so we know we only need to visit // one target. diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 4d9a198eeb2..1411d9be223 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -41,7 +41,10 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { should_cleanup = true; continue; } - if SimplifyToExp::default().simplify(tcx, body, bb_idx, param_env).is_some() { + // unsound: https://github.com/rust-lang/rust/issues/124150 + if tcx.sess.opts.unstable_opts.unsound_mir_opts + && SimplifyToExp::default().simplify(tcx, body, bb_idx, param_env).is_some() + { should_cleanup = true; continue; } @@ -369,8 +372,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { } fn int_equal(l: ScalarInt, r: impl Into<u128>, size: Size) -> bool { - l.try_to_int(l.size()).unwrap() - == ScalarInt::try_from_uint(r, size).unwrap().try_to_int(size).unwrap() + l.assert_int(l.size()) == ScalarInt::try_from_uint(r, size).unwrap().assert_int(size) } // We first compare the two branches, and then the other branches need to fulfill the same conditions. diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index a9d4b860b7a..1f4af0ec63d 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -490,14 +490,14 @@ impl<'tcx> Validator<'_, 'tcx> { } _ => None, }; - match rhs_val.map(|x| x.try_to_uint(sz).unwrap()) { + match rhs_val.map(|x| x.assert_uint(sz)) { // for the zero test, int vs uint does not matter Some(x) if x != 0 => {} // okay _ => return Err(Unpromotable), // value not known or 0 -- not okay } // Furthermore, for signed divison, we also have to exclude `int::MIN / -1`. if lhs_ty.is_signed() { - match rhs_val.map(|x| x.try_to_int(sz).unwrap()) { + match rhs_val.map(|x| x.assert_int(sz)) { Some(-1) | None => { // The RHS is -1 or unknown, so we have to be careful. // But is the LHS int::MIN? @@ -508,7 +508,7 @@ impl<'tcx> Validator<'_, 'tcx> { _ => None, }; let lhs_min = sz.signed_int_min(); - match lhs_val.map(|x| x.try_to_int(sz).unwrap()) { + match lhs_val.map(|x| x.assert_int(sz)) { Some(x) if x != lhs_min => {} // okay _ => return Err(Unpromotable), // value not known or int::MIN -- not okay } diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 1c1ca0bac81..1abb1d29562 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -30,7 +30,7 @@ use unescape_error_reporting::{emit_unescape_error, escaped_char}; // // This assertion is in this crate, rather than in `rustc_lexer`, because that // crate cannot depend on `rustc_data_structures`. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(rustc_lexer::Token, 12); #[derive(Clone, Debug)] @@ -204,6 +204,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> { self.ident(start) } rustc_lexer::TokenKind::InvalidIdent + | rustc_lexer::TokenKind::InvalidPrefix // Do not recover an identifier with emoji if the codepoint is a confusable // with a recoverable substitution token, like `➖`. if !UNICODE_ARRAY @@ -301,7 +302,9 @@ impl<'psess, 'src> StringReader<'psess, 'src> { rustc_lexer::TokenKind::Caret => token::BinOp(token::Caret), rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent), - rustc_lexer::TokenKind::Unknown | rustc_lexer::TokenKind::InvalidIdent => { + rustc_lexer::TokenKind::Unknown + | rustc_lexer::TokenKind::InvalidIdent + | rustc_lexer::TokenKind::InvalidPrefix => { // Don't emit diagnostics for sequences of the same invalid token if swallow_next_invalid > 0 { swallow_next_invalid -= 1; diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index baaed5ec37b..62c8f9f5dac 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -454,7 +454,7 @@ fn make_token_stream( } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index a4a9ba9d229..4a996f89a9a 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -179,7 +179,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_data_structures::static_assert_size!(Parser<'_>, 264); /// Stores span information about a closure. diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs index ccda43c827c..faf6ca78467 100644 --- a/compiler/rustc_parse_format/src/lib.rs +++ b/compiler/rustc_parse_format/src/lib.rs @@ -1087,7 +1087,7 @@ fn unescape_string(string: &str) -> Option<string::String> { } // Assert a reasonable size for `Piece` -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] rustc_index::static_assert_size!(Piece<'_>, 16); #[cfg(test)] diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 2a7b5650fc2..d5b22f841d2 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -148,6 +148,8 @@ pub enum InstrumentCoverage { pub struct CoverageOptions { /// Add branch coverage instrumentation. pub branch: bool, + /// Add mcdc coverage instrumentation. + pub mcdc: bool, } /// Settings for `-Z instrument-xray` flag. @@ -2880,7 +2882,7 @@ pub(crate) mod dep_tracking { use rustc_feature::UnstableFeatures; use rustc_span::edition::Edition; use rustc_span::RealFileName; - use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel}; + use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel, WasmCAbi}; use rustc_target::spec::{ RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TargetTriple, TlsModel, }; @@ -2978,6 +2980,7 @@ pub(crate) mod dep_tracking { Polonius, InliningThreshold, FunctionReturn, + WasmCAbi, ); impl<T1, T2> DepTrackingHash for (T1, T2) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index c4d802a222b..d5108058948 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -8,7 +8,9 @@ use rustc_data_structures::profiling::TimePassesFormat; use rustc_data_structures::stable_hasher::Hash64; use rustc_errors::ColorConfig; use rustc_errors::{LanguageIdentifier, TerminalUrl}; -use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet}; +use rustc_target::spec::{ + CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet, WasmCAbi, +}; use rustc_target::spec::{ RelocModel, RelroLevel, SplitDebuginfo, StackProtector, TargetTriple, TlsModel, }; @@ -396,7 +398,7 @@ mod desc { pub const parse_optimization_fuel: &str = "crate=integer"; pub const parse_dump_mono_stats: &str = "`markdown` (default) or `json`"; pub const parse_instrument_coverage: &str = parse_bool; - pub const parse_coverage_options: &str = "`branch` or `no-branch`"; + pub const parse_coverage_options: &str = "either `no-branch`, `branch` or `mcdc`"; pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`"; pub const parse_unpretty: &str = "`string` or `string=string`"; pub const parse_treat_err_as_bug: &str = "either no value or a non-negative number"; @@ -441,6 +443,7 @@ mod desc { "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number"; pub const parse_llvm_module_flag: &str = "<key>:<type>:<value>:<behavior>. Type must currently be `u32`. Behavior should be one of (`error`, `warning`, `require`, `override`, `append`, `appendunique`, `max`, `min`)"; pub const parse_function_return: &str = "`keep` or `thunk-extern`"; + pub const parse_wasm_c_abi: &str = "`legacy` or `spec`"; } mod parse { @@ -946,17 +949,19 @@ mod parse { let Some(v) = v else { return true }; for option in v.split(',') { - let (option, enabled) = match option.strip_prefix("no-") { - Some(without_no) => (without_no, false), - None => (option, true), - }; - let slot = match option { - "branch" => &mut slot.branch, + match option { + "no-branch" => { + slot.branch = false; + slot.mcdc = false; + } + "branch" => slot.branch = true, + "mcdc" => { + slot.branch = true; + slot.mcdc = true; + } _ => return false, - }; - *slot = enabled; + } } - true } @@ -1433,6 +1438,15 @@ mod parse { } true } + + pub(crate) fn parse_wasm_c_abi(slot: &mut WasmCAbi, v: Option<&str>) -> bool { + match v { + Some("spec") => *slot = WasmCAbi::Spec, + Some("legacy") => *slot = WasmCAbi::Legacy, + _ => return false, + } + true + } } options! { @@ -1717,6 +1731,9 @@ options! { "enable MIR inlining (default: no)"), inline_mir_hint_threshold: Option<usize> = (None, parse_opt_number, [TRACKED], "inlining threshold for functions with inline hint (default: 100)"), + inline_mir_preserve_debug: Option<bool> = (None, parse_opt_bool, [TRACKED], + "when MIR inlining, whether to preserve debug info for callee variables \ + (default: preserve for debuginfo != None, otherwise remove)"), inline_mir_threshold: Option<usize> = (None, parse_opt_number, [TRACKED], "a default MIR inlining threshold (default: 50)"), input_stats: bool = (false, parse_bool, [UNTRACKED], @@ -2058,6 +2075,8 @@ written to standard error output)"), Requires `-Clto[=[fat,yes]]`"), wasi_exec_model: Option<WasiExecModel> = (None, parse_wasi_exec_model, [TRACKED], "whether to build a wasi command or reactor"), + wasm_c_abi: WasmCAbi = (WasmCAbi::Legacy, parse_wasm_c_abi, [TRACKED], + "use spec-compliant C ABI for `wasm32-unknown-unknown` (default: legacy)"), write_long_types_to_disk: bool = (true, parse_bool, [UNTRACKED], "whether long type names should be written to files instead of being printed in errors"), // tidy-alphabetical-end diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index b63c119eee0..2bc14b43234 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -352,6 +352,10 @@ impl Session { self.instrument_coverage() && self.opts.unstable_opts.coverage_options.branch } + pub fn instrument_coverage_mcdc(&self) -> bool { + self.instrument_coverage() && self.opts.unstable_opts.coverage_options.mcdc + } + pub fn is_sanitizer_cfi_enabled(&self) -> bool { self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI) } diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 4502df339d1..cdd82aa9dbc 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -1,6 +1,6 @@ use crate::abi::{self, Abi, Align, FieldsShape, Size}; use crate::abi::{HasDataLayout, TyAbiInterface, TyAndLayout}; -use crate::spec::{self, HasTargetSpec}; +use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt}; use rustc_span::Symbol; use std::fmt; use std::str::FromStr; @@ -829,7 +829,7 @@ impl<'a, Ty> FnAbi<'a, Ty> { ) -> Result<(), AdjustForForeignAbiError> where Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, + C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt, { if abi == spec::abi::Abi::X86Interrupt { if let Some(arg) = self.args.first_mut() { @@ -886,7 +886,9 @@ impl<'a, Ty> FnAbi<'a, Ty> { "sparc" => sparc::compute_abi_info(cx, self), "sparc64" => sparc64::compute_abi_info(cx, self), "nvptx64" => { - if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::PtxKernel { + if cx.target_spec().adjust_abi(cx, abi, self.c_variadic) + == spec::abi::Abi::PtxKernel + { nvptx64::compute_ptx_kernel_abi_info(cx, self) } else { nvptx64::compute_abi_info(self) @@ -895,7 +897,7 @@ impl<'a, Ty> FnAbi<'a, Ty> { "hexagon" => hexagon::compute_abi_info(self), "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self), "wasm32" | "wasm64" => { - if cx.target_spec().adjust_abi(abi, self.c_variadic) == spec::abi::Abi::Wasm { + if cx.target_spec().adjust_abi(cx, abi, self.c_variadic) == spec::abi::Abi::Wasm { wasm::compute_wasm_abi_info(self) } else { wasm::compute_c_abi_info(cx, self) @@ -947,7 +949,7 @@ impl FromStr for Conv { } // Some types are used a lot. Make sure they don't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] mod size_asserts { use super::*; use rustc_data_structures::static_assert_size; diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 3a69b19ee60..291a761913b 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -37,7 +37,7 @@ use crate::abi::call::Conv; use crate::abi::{Endian, Integer, Size, TargetDataLayout, TargetDataLayoutErrors}; use crate::json::{Json, ToJson}; -use crate::spec::abi::{lookup as lookup_abi, Abi}; +use crate::spec::abi::Abi; use crate::spec::crt_objects::CrtObjects; use rustc_fs_util::try_canonicalize; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; @@ -1915,6 +1915,19 @@ impl HasTargetSpec for Target { } } +/// Which C ABI to use for `wasm32-unknown-unknown`. +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum WasmCAbi { + /// Spec-compliant C ABI. + Spec, + /// Legacy ABI. Which is non-spec-compliant. + Legacy, +} + +pub trait HasWasmCAbiOpt { + fn wasm_c_abi_opt(&self) -> WasmCAbi; +} + type StaticCow<T> = Cow<'static, T>; /// Optional aspects of a target specification. @@ -2273,9 +2286,6 @@ pub struct TargetOptions { /// distributed with the target, the sanitizer should still appear in this list for the target. pub supported_sanitizers: SanitizerSet, - /// If present it's a default value to use for adjusting the C ABI. - pub default_adjusted_cabi: Option<Abi>, - /// Minimum number of bits in #[repr(C)] enum. Defaults to the size of c_int pub c_enum_min_bits: Option<u64>, @@ -2507,7 +2517,6 @@ impl Default for TargetOptions { // `Off` is supported by default, but targets can remove this manually, e.g. Windows. supported_split_debuginfo: Cow::Borrowed(&[SplitDebuginfo::Off]), supported_sanitizers: SanitizerSet::empty(), - default_adjusted_cabi: None, c_enum_min_bits: None, generate_arange_section: true, supports_stack_protector: true, @@ -2538,9 +2547,21 @@ impl DerefMut for Target { impl Target { /// Given a function ABI, turn it into the correct ABI for this target. - pub fn adjust_abi(&self, abi: Abi, c_variadic: bool) -> Abi { + pub fn adjust_abi<C>(&self, cx: &C, abi: Abi, c_variadic: bool) -> Abi + where + C: HasWasmCAbiOpt, + { match abi { - Abi::C { .. } => self.default_adjusted_cabi.unwrap_or(abi), + Abi::C { .. } => { + if self.arch == "wasm32" + && self.os == "unknown" + && cx.wasm_c_abi_opt() == WasmCAbi::Legacy + { + Abi::Wasm + } else { + abi + } + } // On Windows, `extern "system"` behaves like msvc's `__stdcall`. // `__stdcall` only applies on x86 and on non-variadic functions: @@ -3079,16 +3100,6 @@ impl Target { } } } ); - ($key_name:ident, Option<Abi>) => ( { - let name = (stringify!($key_name)).replace("_", "-"); - obj.remove(&name).and_then(|o| o.as_str().and_then(|s| { - match lookup_abi(s) { - Ok(abi) => base.$key_name = Some(abi), - _ => return Some(Err(format!("'{}' is not a valid value for abi", s))), - } - Some(Ok(())) - })).unwrap_or(Ok(())) - } ); ($key_name:ident, TargetFamilies) => ( { if let Some(value) = obj.remove("target-family") { if let Some(v) = value.as_array() { @@ -3238,7 +3249,6 @@ impl Target { key!(split_debuginfo, SplitDebuginfo)?; key!(supported_split_debuginfo, fallible_list)?; key!(supported_sanitizers, SanitizerSet)?; - key!(default_adjusted_cabi, Option<Abi>)?; key!(generate_arange_section, bool); key!(supports_stack_protector, bool); key!(entry_name); @@ -3502,10 +3512,6 @@ impl ToJson for Target { target_option_val!(entry_abi); target_option_val!(supports_xray); - if let Some(abi) = self.default_adjusted_cabi { - d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json()); - } - // Serializing `-Clink-self-contained` needs a dynamic key to support the // backwards-compatible variants. d.insert(self.link_self_contained.json_key().into(), self.link_self_contained.to_json()); diff --git a/compiler/rustc_target/src/spec/targets/wasm32_unknown_unknown.rs b/compiler/rustc_target/src/spec/targets/wasm32_unknown_unknown.rs index e743a18ce80..23f4772c39c 100644 --- a/compiler/rustc_target/src/spec/targets/wasm32_unknown_unknown.rs +++ b/compiler/rustc_target/src/spec/targets/wasm32_unknown_unknown.rs @@ -10,23 +10,12 @@ //! This target is more or less managed by the Rust and WebAssembly Working //! Group nowadays at <https://github.com/rustwasm>. -use crate::spec::abi::Abi; use crate::spec::{base, Cc, LinkerFlavor, Target}; pub fn target() -> Target { let mut options = base::wasm::options(); options.os = "unknown".into(); - // This is a default for backwards-compatibility with the original - // definition of this target oh-so-long-ago. Once the "wasm" ABI is - // stable and the wasm-bindgen project has switched to using it then there's - // no need for this and it can be removed. - // - // Currently this is the reason that this target's ABI is mismatched with - // clang's ABI. This means that, in the limit, you can't merge C and Rust - // code on this target due to this ABI mismatch. - options.default_adjusted_cabi = Some(Abi::Wasm); - options.add_pre_link_args( LinkerFlavor::WasmLld(Cc::No), &[ diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs index 057d00aeae8..d54f714b22c 100644 --- a/compiler/rustc_trait_selection/src/lib.rs +++ b/compiler/rustc_trait_selection/src/lib.rs @@ -31,7 +31,7 @@ #[macro_use] extern crate rustc_macros; -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] #[macro_use] extern crate rustc_data_structures; #[macro_use] diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs index a04a3bc6ebe..fbff78304ac 100644 --- a/compiler/rustc_trait_selection/src/traits/fulfill.rs +++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs @@ -72,7 +72,7 @@ pub struct PendingPredicateObligation<'tcx> { } // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger. -#[cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), target_pointer_width = "64"))] +#[cfg(target_pointer_width = "64")] static_assert_size!(PendingPredicateObligation<'_>, 72); impl<'tcx> FulfillmentContext<'tcx> { diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index 4a8df6c6a5b..67865bfcaa3 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -641,7 +641,7 @@ pub fn compute_inherent_assoc_ty_args<'a, 'b, 'tcx>( ); } - match selcx.infcx.at(&cause, param_env).eq(DefineOpaqueTypes::No, impl_ty, self_ty) { + match selcx.infcx.at(&cause, param_env).eq(DefineOpaqueTypes::Yes, impl_ty, self_ty) { Ok(mut ok) => obligations.append(&mut ok.obligations), Err(_) => { tcx.dcx().span_bug( diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs index 12c984f1603..edd3227210b 100644 --- a/compiler/rustc_transmute/src/layout/tree.rs +++ b/compiler/rustc_transmute/src/layout/tree.rs @@ -420,7 +420,7 @@ pub(crate) mod rustc { fn from_tag(tag: ScalarInt, tcx: TyCtxt<'tcx>) -> Self { use rustc_target::abi::Endian; let size = tag.size(); - let bits = tag.to_bits(size).unwrap(); + let bits = tag.assert_bits(size); let bytes: [u8; 16]; let bytes = match tcx.data_layout.endian { Endian::Little => { diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 65c3cf1a607..f0cea1f0baf 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -322,7 +322,7 @@ fn fn_sig_for_fn_abi<'tcx>( #[inline] fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi, c_variadic: bool) -> Conv { use rustc_target::spec::abi::Abi::*; - match tcx.sess.target.adjust_abi(abi, c_variadic) { + match tcx.sess.target.adjust_abi(&tcx, abi, c_variadic) { RustIntrinsic | Rust | RustCall => Conv::Rust, // This is intentionally not using `Conv::Cold`, as that has to preserve |