Merge commit 'b385428e3ddf330805241e7758e773f933357c4b' into subtree-update_cg_gcc_2024-03-05

1 files changed, 133 insertions, 90 deletions

diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs
index 78e8e32b972..a237f3e6490 100644
--- a/compiler/rustc_codegen_gcc/src/asm.rs
+++ b/compiler/rustc_codegen_gcc/src/asm.rs
@@ -2,7 +2,10 @@ use gccjit::{LValue, RValue, ToRValue, Type};
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::mir::operand::OperandValue;
 use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::traits::{AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef, InlineAsmOperandRef};
+use rustc_codegen_ssa::traits::{
+    AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef,
+    InlineAsmOperandRef,
+};
 
 use rustc_middle::{bug, ty::Instance};
 use rustc_span::Span;
@@ -11,11 +14,10 @@ use rustc_target::asm::*;
 use std::borrow::Cow;
 
 use crate::builder::Builder;
+use crate::callee::get_fn;
 use crate::context::CodegenCx;
 use crate::errors::UnwindingInlineAsm;
 use crate::type_of::LayoutGccExt;
-use crate::callee::get_fn;
-
 
 // Rust asm! and GCC Extended Asm semantics differ substantially.
 //
@@ -68,7 +70,6 @@ use crate::callee::get_fn;
 const ATT_SYNTAX_INS: &str = ".att_syntax noprefix\n\t";
 const INTEL_SYNTAX_INS: &str = "\n\t.intel_syntax noprefix";
 
-
 struct AsmOutOperand<'a, 'tcx, 'gcc> {
     rust_idx: usize,
     constraint: &'a str,
@@ -76,13 +77,13 @@ struct AsmOutOperand<'a, 'tcx, 'gcc> {
     readwrite: bool,
 
     tmp_var: LValue<'gcc>,
-    out_place: Option<PlaceRef<'tcx, RValue<'gcc>>>
+    out_place: Option<PlaceRef<'tcx, RValue<'gcc>>>,
 }
 
 struct AsmInOperand<'a, 'tcx> {
     rust_idx: usize,
     constraint: Cow<'a, str>,
-    val: RValue<'tcx>
+    val: RValue<'tcx>,
 }
 
 impl AsmOutOperand<'_, '_, '_> {
@@ -95,23 +96,28 @@ impl AsmOutOperand<'_, '_, '_> {
             res.push('&');
         }
 
-        res.push_str(&self.constraint);
+        res.push_str(self.constraint);
         res
     }
 }
 
 enum ConstraintOrRegister {
     Constraint(&'static str),
-    Register(&'static str)
+    Register(&'static str),
 }
 
-
 impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+    fn codegen_inline_asm(
+        &mut self,
+        template: &[InlineAsmTemplatePiece],
+        rust_operands: &[InlineAsmOperandRef<'tcx, Self>],
+        options: InlineAsmOptions,
+        span: &[Span],
+        instance: Instance<'_>,
+        _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+    ) {
         if options.contains(InlineAsmOptions::MAY_UNWIND) {
-            self.sess().dcx()
-                .create_err(UnwindingInlineAsm { span: span[0] })
-                .emit();
+            self.sess().dcx().create_err(UnwindingInlineAsm { span: span[0] }).emit();
             return;
         }
 
@@ -157,32 +163,40 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     use ConstraintOrRegister::*;
 
                     let (constraint, ty) = match (reg_to_gcc(reg), place) {
-                        (Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx)),
+                        (Constraint(constraint), Some(place)) => {
+                            (constraint, place.layout.gcc_type(self.cx))
+                        }
                         // When `reg` is a class and not an explicit register but the out place is not specified,
                         // we need to create an unused output variable to assign the output to. This var
                         // needs to be of a type that's "compatible" with the register class, but specific type
                         // doesn't matter.
-                        (Constraint(constraint), None) => (constraint, dummy_output_type(self.cx, reg.reg_class())),
+                        (Constraint(constraint), None) => {
+                            (constraint, dummy_output_type(self.cx, reg.reg_class()))
+                        }
                         (Register(_), Some(_)) => {
                             // left for the next pass
-                            continue
-                        },
+                            continue;
+                        }
                         (Register(reg_name), None) => {
                             // `clobber_abi` can add lots of clobbers that are not supported by the target,
                             // such as AVX-512 registers, so we just ignore unsupported registers
-                            let is_target_supported = reg.reg_class().supported_types(asm_arch).iter()
-                                .any(|&(_, feature)| {
-                                    if let Some(feature) = feature {
-                                        self.tcx.asm_target_features(instance.def_id()).contains(&feature)
-                                    } else {
-                                        true // Register class is unconditionally supported
-                                    }
-                                });
+                            let is_target_supported =
+                                reg.reg_class().supported_types(asm_arch).iter().any(
+                                    |&(_, feature)| {
+                                        if let Some(feature) = feature {
+                                            self.tcx
+                                                .asm_target_features(instance.def_id())
+                                                .contains(&feature)
+                                        } else {
+                                            true // Register class is unconditionally supported
+                                        }
+                                    },
+                                );
 
                             if is_target_supported && !clobbers.contains(&reg_name) {
                                 clobbers.push(reg_name);
                             }
-                            continue
+                            continue;
                         }
                     };
 
@@ -193,7 +207,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         late,
                         readwrite: false,
                         tmp_var,
-                        out_place: place
+                        out_place: place,
                     });
                 }
 
@@ -202,23 +216,22 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         inputs.push(AsmInOperand {
                             constraint: Cow::Borrowed(constraint),
                             rust_idx,
-                            val: value.immediate()
+                            val: value.immediate(),
                         });
-                    }
-                    else {
+                    } else {
                         // left for the next pass
-                        continue
+                        continue;
                     }
                 }
 
                 InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
-                    let constraint = if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) {
-                        constraint
-                    }
-                    else {
-                        // left for the next pass
-                        continue
-                    };
+                    let constraint =
+                        if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) {
+                            constraint
+                        } else {
+                            // left for the next pass
+                            continue;
+                        };
 
                     // Rustc frontend guarantees that input and output types are "compatible",
                     // so we can just use input var's type for the output variable.
@@ -249,7 +262,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         inputs.push(AsmInOperand {
                             constraint,
                             rust_idx,
-                            val: in_value.immediate()
+                            val: in_value.immediate(),
                         });
                     }
                 }
@@ -267,7 +280,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::SymStatic { def_id } => {
                     // TODO(@Amanieu): Additional mangling is needed on
                     // some targets to add a leading underscore (Mach-O).
-                    constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
+                    constants_len +=
+                        self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
             }
         }
@@ -280,10 +294,9 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
                         let out_place = if let Some(place) = place {
                             place
-                        }
-                        else {
+                        } else {
                             // processed in the previous pass
-                            continue
+                            continue;
                         };
 
                         let ty = out_place.layout.gcc_type(self.cx);
@@ -291,12 +304,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         tmp_var.set_register_name(reg_name);
 
                         outputs.push(AsmOutOperand {
-                            constraint: "r".into(),
+                            constraint: "r",
                             rust_idx,
                             late,
                             readwrite: false,
                             tmp_var,
-                            out_place: Some(out_place)
+                            out_place: Some(out_place),
                         });
                     }
 
@@ -314,7 +327,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         inputs.push(AsmInOperand {
                             constraint: "r".into(),
                             rust_idx,
-                            val: reg_var.to_rvalue()
+                            val: reg_var.to_rvalue(),
                         });
                     }
 
@@ -330,7 +343,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         tmp_var.set_register_name(reg_name);
 
                         outputs.push(AsmOutOperand {
-                            constraint: "r".into(),
+                            constraint: "r",
                             rust_idx,
                             late,
                             readwrite: false,
@@ -342,7 +355,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         inputs.push(AsmInOperand {
                             constraint,
                             rust_idx,
-                            val: in_value.immediate()
+                            val: in_value.immediate(),
                         });
                     }
 
@@ -373,7 +386,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
 
         // 3. Build the template string
 
-        let mut template_str = String::with_capacity(estimate_template_length(template, constants_len, att_dialect));
+        let mut template_str =
+            String::with_capacity(estimate_template_length(template, constants_len, att_dialect));
         if att_dialect {
             template_str.push_str(ATT_SYNTAX_INS);
         }
@@ -383,16 +397,15 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmTemplatePiece::String(ref string) => {
                     for char in string.chars() {
                         // TODO(antoyo): might also need to escape | if rustc doesn't do it.
-                        let escaped_char =
-                            match char {
-                                '%' => "%%",
-                                '{' => "%{",
-                                '}' => "%}",
-                                _ => {
-                                    template_str.push(char);
-                                    continue;
-                                },
-                            };
+                        let escaped_char = match char {
+                            '%' => "%%",
+                            '{' => "%{",
+                            '}' => "%}",
+                            _ => {
+                                template_str.push(char);
+                                continue;
+                            }
+                        };
                         template_str.push_str(escaped_char);
                     }
                 }
@@ -408,9 +421,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     };
 
                     match rust_operands[operand_idx] {
-                        InlineAsmOperandRef::Out { reg, ..  } => {
+                        InlineAsmOperandRef::Out { reg, .. } => {
                             let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
-                            let gcc_index = outputs.iter()
+                            let gcc_index = outputs
+                                .iter()
                                 .position(|op| operand_idx == op.rust_idx)
                                 .expect("wrong rust index");
                             push_to_template(modifier, gcc_index);
@@ -418,7 +432,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
 
                         InlineAsmOperandRef::In { reg, .. } => {
                             let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
-                            let in_gcc_index = inputs.iter()
+                            let in_gcc_index = inputs
+                                .iter()
                                 .position(|op| operand_idx == op.rust_idx)
                                 .expect("wrong rust index");
                             let gcc_index = in_gcc_index + outputs.len();
@@ -429,7 +444,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                             let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
 
                             // The input register is tied to the output, so we can just use the index of the output register
-                            let gcc_index = outputs.iter()
+                            let gcc_index = outputs
+                                .iter()
                                 .position(|op| operand_idx == op.rust_idx)
                                 .expect("wrong rust index");
                             push_to_template(modifier, gcc_index);
@@ -496,7 +512,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         }
         if options.contains(InlineAsmOptions::NORETURN) {
             let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
-            let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
+            let builtin_unreachable: RValue<'gcc> =
+                unsafe { std::mem::transmute(builtin_unreachable) };
             self.call(self.type_void(), None, None, builtin_unreachable, &[], None);
         }
 
@@ -517,19 +534,23 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 }
 
-fn estimate_template_length(template: &[InlineAsmTemplatePiece], constants_len: usize, att_dialect: bool) -> usize {
-    let len: usize = template.iter().map(|piece| {
-        match *piece {
-            InlineAsmTemplatePiece::String(ref string) => {
-                string.len()
-            }
-            InlineAsmTemplatePiece::Placeholder { .. } => {
-                // '%' + 1 char modifier + 1 char index
-                3
+fn estimate_template_length(
+    template: &[InlineAsmTemplatePiece],
+    constants_len: usize,
+    att_dialect: bool,
+) -> usize {
+    let len: usize = template
+        .iter()
+        .map(|piece| {
+            match *piece {
+                InlineAsmTemplatePiece::String(ref string) => string.len(),
+                InlineAsmTemplatePiece::Placeholder { .. } => {
+                    // '%' + 1 char modifier + 1 char index
+                    3
+                }
             }
-        }
-    })
-    .sum();
+        })
+        .sum();
 
     // increase it by 5% to account for possible '%' signs that'll be duplicated
     // I pulled the number out of blue, but should be fair enough
@@ -562,7 +583,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
 
                 _ => unimplemented!(),
             }
-        },
+        }
         // They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html
         InlineAsmRegOrRegClass::RegClass(reg) => match reg {
             InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
@@ -610,7 +631,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
             | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
                 unreachable!("clobber-only")
-            },
+            }
             InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
@@ -637,7 +658,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
             InlineAsmRegClass::Err => unreachable!(),
-        }
+        },
     };
 
     ConstraintOrRegister::Constraint(constraint)
@@ -653,7 +674,7 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
             unimplemented!()
         }
-        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)=> cx.type_i32(),
+        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
         | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
         InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
@@ -686,7 +707,7 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
         | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
             unreachable!("clobber-only")
-        },
+        }
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
         InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => cx.type_f32(),
@@ -704,9 +725,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
         InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
             bug!("LLVM backend does not support SPIR-V")
-        },
+        }
         InlineAsmRegClass::S390x(
-            S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr
+            S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
         ) => cx.type_i32(),
         InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
         InlineAsmRegClass::Err => unreachable!(),
@@ -714,7 +735,13 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
 }
 
 impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
-    fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) {
+    fn codegen_global_asm(
+        &self,
+        template: &[InlineAsmTemplatePiece],
+        operands: &[GlobalAsmOperandRef<'tcx>],
+        options: InlineAsmOptions,
+        _line_spans: &[Span],
+    ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
         // Default to Intel syntax on x86
@@ -732,15 +759,17 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                     let mut index = 0;
                     while index < string.len() {
                         // NOTE: gcc does not allow inline comment, so remove them.
-                        let comment_index = string[index..].find("//")
+                        let comment_index = string[index..]
+                            .find("//")
                             .map(|comment_index| comment_index + index)
                             .unwrap_or(string.len());
                         template_str.push_str(&string[index..comment_index]);
-                        index = string[comment_index..].find('\n')
+                        index = string[comment_index..]
+                            .find('\n')
                             .map(|index| index + comment_index)
                             .unwrap_or(string.len());
                     }
-                },
+                }
                 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
                     match operands[operand_idx] {
                         GlobalAsmOperandRef::Const { ref string } => {
@@ -782,14 +811,22 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     }
 }
 
-fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>) -> Option<char> {
+fn modifier_to_gcc(
+    arch: InlineAsmArch,
+    reg: InlineAsmRegClass,
+    modifier: Option<char>,
+) -> Option<char> {
     // The modifiers can be retrieved from
     // https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers
     match reg {
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
         | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
-            if modifier == Some('v') { None } else { modifier }
+            if modifier == Some('v') {
+                None
+            } else {
+                modifier
+            }
         }
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
             unreachable!("clobber-only")
@@ -821,7 +858,13 @@ fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option
         }
         InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
         | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
-            None => if arch == InlineAsmArch::X86_64 { Some('q') } else { Some('k') },
+            None => {
+                if arch == InlineAsmArch::X86_64 {
+                    Some('q')
+                } else {
+                    Some('k')
+                }
+            }
             Some('l') => Some('b'),
             Some('h') => Some('h'),
             Some('x') => Some('w'),