Rollup merge of #73347 - tmiasko:incompatible-sanitizers, r=nikic

Diagnose use of incompatible sanitizers

Emit an error when incompatible sanitizer are configured through command
line options. Previously the last one configured prevailed and others
were silently ignored.

Additionally use a set to represent configured sanitizers, making it
possible to enable multiple sanitizers at once. At least in principle,
since currently all of them are considered to be incompatible with
others.

4 files changed, 46 insertions, 56 deletions

diff --git a/src/librustc_codegen_llvm/attributes.rs b/src/librustc_codegen_llvm/attributes.rs
index a4e17a5f675..6234ade8a16 100644
--- a/src/librustc_codegen_llvm/attributes.rs
+++ b/src/librustc_codegen_llvm/attributes.rs
@@ -11,7 +11,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::config::{OptLevel, Sanitizer};
+use rustc_session::config::{OptLevel, SanitizerSet};
 use rustc_session::Session;
 
 use crate::attributes;
@@ -45,26 +45,16 @@ fn inline(cx: &CodegenCx<'ll, '_>, val: &'ll Value, inline: InlineAttr) {
 
 /// Apply LLVM sanitize attributes.
 #[inline]
-pub fn sanitize(cx: &CodegenCx<'ll, '_>, codegen_fn_flags: CodegenFnAttrFlags, llfn: &'ll Value) {
-    if let Some(ref sanitizer) = cx.tcx.sess.opts.debugging_opts.sanitizer {
-        match *sanitizer {
-            Sanitizer::Address => {
-                if !codegen_fn_flags.contains(CodegenFnAttrFlags::NO_SANITIZE_ADDRESS) {
-                    llvm::Attribute::SanitizeAddress.apply_llfn(Function, llfn);
-                }
-            }
-            Sanitizer::Memory => {
-                if !codegen_fn_flags.contains(CodegenFnAttrFlags::NO_SANITIZE_MEMORY) {
-                    llvm::Attribute::SanitizeMemory.apply_llfn(Function, llfn);
-                }
-            }
-            Sanitizer::Thread => {
-                if !codegen_fn_flags.contains(CodegenFnAttrFlags::NO_SANITIZE_THREAD) {
-                    llvm::Attribute::SanitizeThread.apply_llfn(Function, llfn);
-                }
-            }
-            Sanitizer::Leak => {}
-        }
+pub fn sanitize(cx: &CodegenCx<'ll, '_>, no_sanitize: SanitizerSet, llfn: &'ll Value) {
+    let enabled = cx.tcx.sess.opts.debugging_opts.sanitizer - no_sanitize;
+    if enabled.contains(SanitizerSet::ADDRESS) {
+        llvm::Attribute::SanitizeAddress.apply_llfn(Function, llfn);
+    }
+    if enabled.contains(SanitizerSet::MEMORY) {
+        llvm::Attribute::SanitizeMemory.apply_llfn(Function, llfn);
+    }
+    if enabled.contains(SanitizerSet::THREAD) {
+        llvm::Attribute::SanitizeThread.apply_llfn(Function, llfn);
     }
 }
 
@@ -123,9 +113,14 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
     // Currently stack probes seem somewhat incompatible with the address
     // sanitizer and thread sanitizer. With asan we're already protected from
     // stack overflow anyway so we don't really need stack probes regardless.
-    match cx.sess().opts.debugging_opts.sanitizer {
-        Some(Sanitizer::Address | Sanitizer::Thread) => return,
-        _ => {}
+    if cx
+        .sess()
+        .opts
+        .debugging_opts
+        .sanitizer
+        .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD)
+    {
+        return;
     }
 
     // probestack doesn't play nice either with `-C profile-generate`.
@@ -296,7 +291,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) {
         Attribute::NoAlias.apply_llfn(llvm::AttributePlace::ReturnValue, llfn);
     }
-    sanitize(cx, codegen_fn_attrs.flags, llfn);
+    sanitize(cx, codegen_fn_attrs.no_sanitize, llfn);
 
     // Always annotate functions with the target-cpu they are compiled for.
     // Without this, ThinLTO won't inline Rust functions into Clang generated
diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs
index 26f5334668b..868ce876a81 100644
--- a/src/librustc_codegen_llvm/back/write.rs
+++ b/src/librustc_codegen_llvm/back/write.rs
@@ -21,7 +21,7 @@ use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::{self, Lto, OutputType, Passes, Sanitizer, SwitchWithOptPath};
+use rustc_session::config::{self, Lto, OutputType, Passes, SanitizerSet, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::InnerSpan;
 use rustc_target::spec::{CodeModel, RelocModel};
@@ -394,12 +394,13 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
     let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
     // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
     let sanitizer_options = if !is_lto {
-        config.sanitizer.as_ref().map(|s| llvm::SanitizerOptions {
-            sanitize_memory: *s == Sanitizer::Memory,
-            sanitize_thread: *s == Sanitizer::Thread,
-            sanitize_address: *s == Sanitizer::Address,
-            sanitize_recover: config.sanitizer_recover.contains(s),
+        Some(llvm::SanitizerOptions {
+            sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS),
+            sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS),
+            sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY),
+            sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY),
             sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
+            sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
         })
     } else {
         None
@@ -600,25 +601,18 @@ pub(crate) unsafe fn optimize(
 }
 
 unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) {
-    let sanitizer = match &config.sanitizer {
-        None => return,
-        Some(s) => s,
-    };
-
-    let recover = config.sanitizer_recover.contains(sanitizer);
-    match sanitizer {
-        Sanitizer::Address => {
-            passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover));
-            passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover));
-        }
-        Sanitizer::Memory => {
-            let track_origins = config.sanitizer_memory_track_origins as c_int;
-            passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover));
-        }
-        Sanitizer::Thread => {
-            passes.push(llvm::LLVMRustCreateThreadSanitizerPass());
-        }
-        Sanitizer::Leak => {}
+    if config.sanitizer.contains(SanitizerSet::ADDRESS) {
+        let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS);
+        passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover));
+        passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover));
+    }
+    if config.sanitizer.contains(SanitizerSet::MEMORY) {
+        let track_origins = config.sanitizer_memory_track_origins as c_int;
+        let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY);
+        passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover));
+    }
+    if config.sanitizer.contains(SanitizerSet::THREAD) {
+        passes.push(llvm::LLVMRustCreateThreadSanitizerPass());
     }
 }
 
diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs
index 3e17a51528e..e99fb8dcae1 100644
--- a/src/librustc_codegen_llvm/base.rs
+++ b/src/librustc_codegen_llvm/base.rs
@@ -29,12 +29,12 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_middle::dep_graph;
-use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::middle::exported_symbols;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::TyCtxt;
-use rustc_session::config::DebugInfo;
+use rustc_session::config::{DebugInfo, SanitizerSet};
 use rustc_span::symbol::Symbol;
 
 use std::ffi::CString;
@@ -132,7 +132,7 @@ pub fn compile_codegen_unit(
             // If this codegen unit contains the main function, also create the
             // wrapper here
             if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx) {
-                attributes::sanitize(&cx, CodegenFnAttrFlags::empty(), entry);
+                attributes::sanitize(&cx, SanitizerSet::empty(), entry);
             }
 
             // Run replace-all-uses-with for statics that need it
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index 372fb17573a..8063d97aa73 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -439,11 +439,12 @@ pub enum OptStage {
 /// LLVMRustSanitizerOptions
 #[repr(C)]
 pub struct SanitizerOptions {
-    pub sanitize_memory: bool,
-    pub sanitize_thread: bool,
     pub sanitize_address: bool,
-    pub sanitize_recover: bool,
+    pub sanitize_address_recover: bool,
+    pub sanitize_memory: bool,
+    pub sanitize_memory_recover: bool,
     pub sanitize_memory_track_origins: c_int,
+    pub sanitize_thread: bool,
 }
 
 /// LLVMRelocMode