Auto merge of #45920 - sunfishcode:trap-on-unreachable, r=Zoxc

Enable TrapUnreachable in LLVM.

This patch enables LLVM's TrapUnreachable flag, which tells it to translate `unreachable` instructions into hardware trap instructions, rather than allowing control flow to "fall through" into whatever code happens to follow it in memory.

This follows up on https://github.com/rust-lang/rust/issues/28728#issuecomment-332581533. For example, for @zackw's testcase [here](https://github.com/rust-lang/rust/issues/42009#issue-228745924), the output function contains a `ud2` instead of no code, so it won't "fall through" into whatever happens to be next in memory.

(I'm also working on the problem of LLVM optimizing away infinite loops, but the patch here is useful independently.)

I tested this patch on a few different codebases, and the code size increase ranged from 0.0% to 0.1%.

7 files changed, 32 insertions, 4 deletions

diff --git a/src/libcompiler_builtins b/src/libcompiler_builtins
-Subproject 0b9844764ea1f99ea11a7917a4f3ba7fd2db775
+Subproject f5532b22b5d741f3ea207b5b07e3e1ca63476f9
diff --git a/src/librustc_back/target/mod.rs b/src/librustc_back/target/mod.rs
index 8fd4aad89c6..d60d6438b4d 100644
--- a/src/librustc_back/target/mod.rs
+++ b/src/librustc_back/target/mod.rs
@@ -435,6 +435,10 @@ pub struct TargetOptions {
 
     /// Default number of codegen units to use in debug mode
     pub default_codegen_units: Option<u64>,
+
+    /// Whether to generate trap instructions in places where optimization would
+    /// otherwise produce control flow that falls through into unrelated memory.
+    pub trap_unreachable: bool,
 }
 
 impl Default for TargetOptions {
@@ -498,6 +502,7 @@ impl Default for TargetOptions {
             stack_probes: false,
             min_global_align: None,
             default_codegen_units: None,
+            trap_unreachable: true,
         }
     }
 }
@@ -739,6 +744,7 @@ impl Target {
         key!(stack_probes, bool);
         key!(min_global_align, Option<u64>);
         key!(default_codegen_units, Option<u64>);
+        key!(trap_unreachable, bool);
 
         if let Some(array) = obj.find("abi-blacklist").and_then(Json::as_array) {
             for name in array.iter().filter_map(|abi| abi.as_string()) {
@@ -932,6 +938,7 @@ impl ToJson for Target {
         target_option_val!(stack_probes);
         target_option_val!(min_global_align);
         target_option_val!(default_codegen_units);
+        target_option_val!(trap_unreachable);
 
         if default.abi_blacklist != self.options.abi_blacklist {
             d.insert("abi-blacklist".to_string(), self.options.abi_blacklist.iter()
diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs
index ac0e4dde0c1..24c3963fbc4 100644
--- a/src/librustc_llvm/ffi.rs
+++ b/src/librustc_llvm/ffi.rs
@@ -1605,7 +1605,8 @@ extern "C" {
                                        UseSoftFP: bool,
                                        PositionIndependentExecutable: bool,
                                        FunctionSections: bool,
-                                       DataSections: bool)
+                                       DataSections: bool,
+                                       TrapUnreachable: bool)
                                        -> TargetMachineRef;
     pub fn LLVMRustDisposeTargetMachine(T: TargetMachineRef);
     pub fn LLVMRustAddAnalysisPasses(T: TargetMachineRef, PM: PassManagerRef, M: ModuleRef);
diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs
index d59d8ca1a78..e443f13a7a1 100644
--- a/src/librustc_trans/back/write.rs
+++ b/src/librustc_trans/back/write.rs
@@ -196,6 +196,7 @@ pub fn target_machine_factory(sess: &Session)
     let cpu = CString::new(cpu.as_bytes()).unwrap();
     let features = CString::new(target_feature(sess).as_bytes()).unwrap();
     let is_pie_binary = is_pie_binary(sess);
+    let trap_unreachable = sess.target.target.options.trap_unreachable;
 
     Arc::new(move || {
         let tm = unsafe {
@@ -208,6 +209,7 @@ pub fn target_machine_factory(sess: &Session)
                 is_pie_binary,
                 ffunction_sections,
                 fdata_sections,
+                trap_unreachable,
             )
         };
 
diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp
index b397ad1e98f..b4116c96ba1 100644
--- a/src/rustllvm/PassWrapper.cpp
+++ b/src/rustllvm/PassWrapper.cpp
@@ -366,7 +366,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     LLVMRustCodeModel RustCM, LLVMRustRelocMode RustReloc,
     LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat,
     bool PositionIndependentExecutable, bool FunctionSections,
-    bool DataSections) {
+    bool DataSections, bool TrapUnreachable) {
 
   auto CM = fromRust(RustCM);
   auto OptLevel = fromRust(RustOptLevel);
@@ -398,6 +398,14 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   Options.DataSections = DataSections;
   Options.FunctionSections = FunctionSections;
 
+  if (TrapUnreachable) {
+    // Tell LLVM to translate `unreachable` into an explicit trap instruction.
+    // This limits the extent of possible undefined behavior in some cases, as
+    // it prevents control flow from "falling through" into whatever code
+    // happens to be laid out next in memory.
+    Options.TrapUnreachable = true;
+  }
+
   TargetMachine *TM = TheTarget->createTargetMachine(
       Trip.getTriple(), RealCPU, Feature, Options, RM, CM, OptLevel);
   return wrap(TM);
diff --git a/src/test/run-make/intrinsic-unreachable/exit-ret.rs b/src/test/run-make/intrinsic-unreachable/exit-ret.rs
index f5be5a055c3..1b8b644dd78 100644
--- a/src/test/run-make/intrinsic-unreachable/exit-ret.rs
+++ b/src/test/run-make/intrinsic-unreachable/exit-ret.rs
@@ -11,10 +11,15 @@
 #![feature(asm)]
 #![crate_type="lib"]
 
-pub fn exit(n: usize) {
+#[deny(unreachable_code)]
+pub fn exit(n: usize) -> i32 {
     unsafe {
         // Pretend this asm is an exit() syscall.
         asm!("" :: "r"(n) :: "volatile");
         // Can't actually reach this point, but rustc doesn't know that.
     }
+    // This return value is just here to generate some extra code for a return
+    // value, making it easier for the test script to detect whether the
+    // compiler deleted it.
+    42
 }
diff --git a/src/test/run-make/intrinsic-unreachable/exit-unreachable.rs b/src/test/run-make/intrinsic-unreachable/exit-unreachable.rs
index f58d2cd8f91..de63809ab66 100644
--- a/src/test/run-make/intrinsic-unreachable/exit-unreachable.rs
+++ b/src/test/run-make/intrinsic-unreachable/exit-unreachable.rs
@@ -13,10 +13,15 @@
 
 use std::intrinsics;
 
-pub fn exit(n: usize) -> ! {
+#[allow(unreachable_code)]
+pub fn exit(n: usize) -> i32 {
     unsafe {
         // Pretend this asm is an exit() syscall.
         asm!("" :: "r"(n) :: "volatile");
         intrinsics::unreachable()
     }
+    // This return value is just here to generate some extra code for a return
+    // value, making it easier for the test script to detect whether the
+    // compiler deleted it.
+    42
 }