From b9746ce03901fc39707c8c2d0405caf5384b4e97 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Fri, 3 Mar 2023 10:14:57 +0800
Subject: introduce `DynSend` and `DynSync` auto trait

---
 compiler/rustc_data_structures/src/lib.rs    |   5 +-
 compiler/rustc_data_structures/src/marker.rs | 268 +++++++++++++++++++++++++++
 compiler/rustc_data_structures/src/sync.rs   | 233 +++++++++++++++++++----
 3 files changed, 467 insertions(+), 39 deletions(-)
 create mode 100644 compiler/rustc_data_structures/src/marker.rs

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 004017ec5f3..f543e0d4a7a 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -26,9 +26,11 @@
 #![feature(test)]
 #![feature(thread_id_value)]
 #![feature(vec_into_raw_parts)]
+#![feature(allocator_api)]
 #![feature(get_mut_unchecked)]
 #![feature(lint_reasons)]
-#![feature(unwrap_infallible)]
+#![feature(unwrap_infallible)]#![feature(const_mut_refs)]
+#![feature(const_trait_impl)]
 #![feature(strict_provenance)]
 #![feature(ptr_alignment_type)]
 #![feature(macro_metavar_expr)]
@@ -77,6 +79,7 @@ pub mod sorted_map;
 pub mod stable_hasher;
 mod atomic_ref;
 pub mod fingerprint;
+pub mod marker;
 pub mod profiling;
 pub mod sharded;
 pub mod stack;
diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
new file mode 100644
index 00000000000..fef8177214b
--- /dev/null
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -0,0 +1,268 @@
+cfg_if!(
+    if #[cfg(not(parallel_compiler))] {
+        pub auto trait DynSend {}
+        pub auto trait DynSync {}
+
+        impl<T> DynSend for T {}
+        impl<T> DynSync for T {}
+    } else {
+        #[rustc_on_unimplemented(
+            message = "`{Self}` doesn't implement `DynSend`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`",
+            label = "`{Self}` doesn't implement `DynSend`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`"
+        )]
+        // Ensure data structures is `Send` if `sync::active()` is true.
+        // `sync::active()` should be checked before using these data structures.
+        // Note: Ensure that the data structure **will not break**
+        // thread safety after being created.
+        //
+        // `sync::active()` should be checked when downcasting these data structures
+        // to `Send` via `FromDyn`.
+        pub unsafe auto trait DynSend {}
+
+        #[rustc_on_unimplemented(
+            message = "`{Self}` doesn't implement `DynSync`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`",
+            label = "`{Self}` doesn't implement `DynSync`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`"
+        )]
+        // Ensure data structures is `Sync` if `sync::active()` is true.
+        // Note: Ensure that the data structure **will not break**
+        // thread safety after being checked.
+        //
+        // `sync::active()` should be checked when downcasting these data structures
+        // to `Send` via `FromDyn`.
+        pub unsafe auto trait DynSync {}
+
+        // Same with `Sync` and `Send`.
+        unsafe impl<T: DynSync + ?Sized> DynSend for &T {}
+
+        macro_rules! impls_dyn_send_neg {
+            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+                $(impl$(<$($generics1)*>)? !DynSend for $t1 {})*
+            };
+        }
+
+        // Consistent with `std`
+        impls_dyn_send_neg!(
+            [std::env::Args]
+            [std::env::ArgsOs]
+            [*const T where T: ?Sized]
+            [*mut T where T: ?Sized]
+            [std::ptr::NonNull<T> where T: ?Sized]
+            [std::rc::Rc<T> where T: ?Sized]
+            [std::rc::Weak<T> where T: ?Sized]
+            [std::sync::MutexGuard<'_, T> where T: ?Sized]
+            [std::sync::RwLockReadGuard<'_, T> where T: ?Sized]
+            [std::sync::RwLockWriteGuard<'_, T> where T: ?Sized]
+            [std::io::StdoutLock<'_>]
+            [std::io::StderrLock<'_>]
+        );
+        cfg_if!(
+            // Consistent with `std`
+            // `os_imp::Env` is `!Send` in these platforms
+            if #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))] {
+                impl !DynSend for std::env::VarsOs {}
+            }
+        );
+
+        macro_rules! already_send {
+            ($([$ty: ty])*) => {
+                $(unsafe impl DynSend for $ty where $ty: Send {})*
+            };
+        }
+
+        // These structures are already `Send`.
+        already_send!(
+            [std::backtrace::Backtrace]
+            [std::io::Stdout]
+            [std::io::Stderr]
+            [std::io::Error]
+            [std::fs::File]
+            [rustc_arena::DroplessArena]
+            [crate::memmap::Mmap]
+            [crate::profiling::SelfProfiler]
+        );
+
+        macro_rules! impl_dyn_send {
+            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+                $(unsafe impl<$($generics2)*> DynSend for $ty {})*
+            };
+        }
+
+        impl_dyn_send!(
+            [std::sync::atomic::AtomicPtr<T> where T]
+            [std::sync::Mutex<T> where T: ?Sized+ DynSend]
+            [std::sync::mpsc::Sender<T> where T: DynSend]
+            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+            [std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
+            [std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
+            [std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+            [std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
+            [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
+            [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
+            [crate::sync::Lock<T> where T: DynSend]
+            [crate::sync::RwLock<T> where T: DynSend]
+            [rustc_arena::TypedArena<T> where T: DynSend]
+            [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
+            [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+            [thin_vec::ThinVec<T> where T: DynSend]
+            [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
+
+            // We use `Send` here to omit some extra code, since they are only
+            // used in `Send` situations now.
+            [crate::owning_ref::OwningRef<O, T> where O: Send, T: ?Sized + Send]
+            [crate::owning_ref::OwningRefMut<O, T> where O: Send, T: ?Sized + Send]
+        );
+
+        macro_rules! impls_dyn_sync_neg {
+            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+                $(impl$(<$($generics1)*>)? !DynSync for $t1 {})*
+            };
+        }
+
+        // Consistent with `std`
+        impls_dyn_sync_neg!(
+            [std::env::Args]
+            [std::env::ArgsOs]
+            [*const T where T: ?Sized]
+            [*mut T where T: ?Sized]
+            [std::cell::Cell<T> where T: ?Sized]
+            [std::cell::RefCell<T> where T: ?Sized]
+            [std::cell::UnsafeCell<T> where T: ?Sized]
+            [std::ptr::NonNull<T> where T: ?Sized]
+            [std::rc::Rc<T> where T: ?Sized]
+            [std::rc::Weak<T> where T: ?Sized]
+            [std::cell::OnceCell<T> where T]
+            [std::sync::mpsc::Receiver<T> where T]
+            [std::sync::mpsc::Sender<T> where T]
+        );
+        cfg_if!(
+            // Consistent with `std`
+            // `os_imp::Env` is `!Sync` in these platforms
+            if #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))] {
+                impl !DynSync for std::env::VarsOs {}
+            }
+        );
+
+        macro_rules! already_sync {
+            ($([$ty: ty])*) => {
+                $(unsafe impl DynSync for $ty where $ty: Sync {})*
+            };
+        }
+
+        // These structures are already `Sync`.
+        already_sync!(
+            [std::sync::atomic::AtomicBool]
+            [std::sync::atomic::AtomicUsize]
+            [std::sync::atomic::AtomicU8]
+            [std::sync::atomic::AtomicU32]
+            [std::sync::atomic::AtomicU64]
+            [std::backtrace::Backtrace]
+            [std::io::Error]
+            [std::fs::File]
+            [jobserver_crate::Client]
+            [crate::memmap::Mmap]
+            [crate::profiling::SelfProfiler]
+        );
+
+        macro_rules! impl_dyn_sync {
+            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+                $(unsafe impl<$($generics2)*> DynSync for $ty {})*
+            };
+        }
+
+        impl_dyn_sync!(
+            [std::sync::atomic::AtomicPtr<T> where T]
+            [std::sync::OnceLock<T> where T: DynSend + DynSync]
+            [std::sync::Mutex<T> where T: ?Sized + DynSend]
+            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+            [std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
+            [std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
+            [std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+            [std::collections::BTreeMap<K, V, A> where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync]
+            [Vec<T, A> where T: DynSync, A: std::alloc::Allocator + DynSync]
+            [Box<T, A> where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync]
+            [crate::sync::Lock<T> where T: DynSend]
+            [crate::sync::RwLock<T> where T: DynSend + DynSync]
+            [crate::sync::OneThread<T> where T]
+            [crate::sync::WorkerLocal<T> where T: DynSend]
+            [crate::intern::Interned<'a, T> where 'a, T: DynSync]
+            [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
+            [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
+            [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
+            [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+            [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
+            [thin_vec::ThinVec<T> where T: DynSync]
+
+            // We use `Sync` here to omit some extra code, since they are only
+            // used in `Sync` situations now.
+            [crate::owning_ref::OwningRef<O, T> where O: Sync, T: ?Sized + Sync]
+            [crate::owning_ref::OwningRefMut<O, T> where O: Sync, T: ?Sized + Sync]
+        );
+    }
+);
+
+pub fn assert_dyn_sync<T: ?Sized + DynSync>() {}
+pub fn assert_dyn_send<T: ?Sized + DynSend>() {}
+pub fn assert_dyn_send_val<T: ?Sized + DynSend>(_t: &T) {}
+pub fn assert_dyn_send_sync_val<T: ?Sized + DynSync + DynSend>(_t: &T) {}
+
+#[derive(Copy, Clone)]
+pub struct FromDyn<T>(T);
+
+impl<T> FromDyn<T> {
+    // Check `sync::active()` when creating this structure
+    // and downcasting to `Send`. So we can ensure it is
+    // thread-safe.
+    #[inline(always)]
+    pub fn from(val: T) -> Self {
+        #[cfg(parallel_compiler)]
+        assert!(crate::sync::active());
+        FromDyn(val)
+    }
+
+    #[inline(always)]
+    pub fn into_inner(self) -> T {
+        self.0
+    }
+}
+
+// `FromDyn` is `Send` if `T` is `DynSend`, since it check when created.
+#[cfg(parallel_compiler)]
+unsafe impl<T: DynSend> Send for FromDyn<T> {}
+
+// `FromDyn` is `Sync` if `T` is `DynSync`, since it check when created.
+#[cfg(parallel_compiler)]
+unsafe impl<T: DynSync> Sync for FromDyn<T> {}
+
+impl<T> const std::ops::Deref for FromDyn<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[derive(Copy, Clone)]
+pub struct IntoDyn<T: ?Sized>(pub T);
+
+#[cfg(parallel_compiler)]
+unsafe impl<T: ?Sized + Send> DynSend for IntoDyn<T> {}
+#[cfg(parallel_compiler)]
+unsafe impl<T: ?Sized + Sync> DynSync for IntoDyn<T> {}
+
+impl<T> const std::ops::Deref for IntoDyn<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.0
+    }
+}
+
+impl<T> const std::ops::DerefMut for IntoDyn<T> {
+    fn deref_mut(&mut self) -> &mut T {
+        &mut self.0
+    }
+}
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index e73ca56efa0..5f865486880 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -39,6 +39,7 @@
 //!
 //! [^2] `MTLockRef` is a typedef.
 
+pub use crate::marker::*;
 use crate::owned_slice::OwnedSlice;
 use std::collections::HashMap;
 use std::hash::{BuildHasher, Hash};
@@ -55,6 +56,37 @@ pub use vec::{AppendOnlyIndexVec, AppendOnlyVec};
 
 mod vec;
 
+mod mode {
+    use super::Ordering;
+    use std::sync::atomic::AtomicU8;
+
+    const UNINITIALIZED: u8 = 0;
+    const INACTIVE: u8 = 1;
+    const ACTIVE: u8 = 2;
+
+    static MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED);
+
+    #[inline]
+    pub fn active() -> bool {
+        match MODE.load(Ordering::Relaxed) {
+            INACTIVE => false,
+            ACTIVE => true,
+            _ => panic!("uninitialized parallel mode!"),
+        }
+    }
+
+    // Only set by the `-Z threads` compile option
+    pub fn set(parallel: bool) {
+        let set: u8 = if parallel { ACTIVE } else { INACTIVE };
+        let previous =
+            MODE.compare_exchange(UNINITIALIZED, set, Ordering::Relaxed, Ordering::Relaxed);
+
+        // Check that the mode was either uninitialized or was already set to the requested mode.
+        assert!(previous.is_ok() || previous == Err(set));
+    }
+}
+
+pub use mode::{active, set};
 cfg_if! {
     if #[cfg(not(parallel_compiler))] {
         pub unsafe auto trait Send {}
@@ -149,7 +181,7 @@ cfg_if! {
 
         #[macro_export]
         macro_rules! parallel {
-            ($($blocks:tt),*) => {
+            ($($blocks:block),*) => {{
                 // We catch panics here ensuring that all the blocks execute.
                 // This makes behavior consistent with the parallel compiler.
                 let mut panic = None;
@@ -165,13 +197,7 @@ cfg_if! {
                 if let Some(panic) = panic {
                     ::std::panic::resume_unwind(panic);
                 }
-            }
-        }
-
-        pub use Iterator as ParallelIterator;
-
-        pub fn par_iter<T: IntoIterator>(t: T) -> T::IntoIter {
-            t.into_iter()
+            }}
         }
 
         pub fn par_for_each_in<T: IntoIterator>(t: T, mut for_each: impl FnMut(T::Item) + Sync + Send) {
@@ -190,6 +216,29 @@ cfg_if! {
             }
         }
 
+        pub fn par_map<T: IntoIterator, R, C: FromIterator<R>>(
+            t: T,
+            mut map: impl FnMut(<<T as IntoIterator>::IntoIter as Iterator>::Item) -> R,
+        ) -> C {
+            // We catch panics here ensuring that all the loop iterations execute.
+            let mut panic = None;
+            let r = t.into_iter().filter_map(|i| {
+                match catch_unwind(AssertUnwindSafe(|| map(i))) {
+                    Ok(r) => Some(r),
+                    Err(p) => {
+                        if panic.is_none() {
+                            panic = Some(p);
+                        }
+                        None
+                    }
+                }
+            }).collect();
+            if let Some(panic) = panic {
+                resume_unwind(panic);
+            }
+            r
+        }
+
         pub type MetadataRef = OwnedSlice;
 
         pub use std::rc::Rc as Lrc;
@@ -302,46 +351,159 @@ cfg_if! {
         use parking_lot::RwLock as InnerRwLock;
 
         use std::thread;
-        pub use rayon::{join, scope};
+
+        #[inline]
+        pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
+        where
+            A: FnOnce() -> RA + DynSend,
+            B: FnOnce() -> RB + DynSend,
+        {
+            if mode::active() {
+                let oper_a = FromDyn::from(oper_a);
+                let oper_b = FromDyn::from(oper_b);
+                let (a, b) = rayon::join(move || FromDyn::from(oper_a.into_inner()()), move || FromDyn::from(oper_b.into_inner()()));
+                (a.into_inner(), b.into_inner())
+            } else {
+                (oper_a(), oper_b())
+            }
+        }
+
+        pub fn scope<'scope, OP, R>(op: OP) -> R
+        where
+            OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
+            R: DynSend,
+        {
+            let op = FromDyn::from(op);
+            rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
+        }
 
         /// Runs a list of blocks in parallel. The first block is executed immediately on
         /// the current thread. Use that for the longest running block.
         #[macro_export]
         macro_rules! parallel {
-            (impl $fblock:tt [$($c:tt,)*] [$block:tt $(, $rest:tt)*]) => {
-                parallel!(impl $fblock [$block, $($c,)*] [$($rest),*])
+            ($fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => {
+                parallel!($fblock [$block, $($c,)*] [$($rest),*])
+            };
+            ($fblock:block [$($blocks:expr,)*] []) => {
+                {
+                    ::rustc_data_structures::sync::scope(|s| {
+                        $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks);
+                        s.spawn(move |_| block.into_inner()());)*
+                        (|| $fblock)();
+                    });
+                }
             };
-            (impl $fblock:tt [$($blocks:tt,)*] []) => {
-                ::rustc_data_structures::sync::scope(|s| {
+            ($fblock:block, $($blocks:block),*) => {
+                if rustc_data_structures::sync::active() {
+                    // Reverse the order of the later blocks since Rayon executes them in reverse order
+                    // when using a single thread. This ensures the execution order matches that
+                    // of a single threaded rustc
+                    parallel!($fblock [] [$($blocks),*]);
+                } else {
+                    // We catch panics here ensuring that all the blocks execute.
+                    // This makes behavior consistent with the parallel compiler.
+                    let mut panic = None;
                     $(
-                        s.spawn(|_| $blocks);
+                        if let Err(p) = ::std::panic::catch_unwind(
+                            ::std::panic::AssertUnwindSafe(|| $blocks)
+                        ) {
+                            if panic.is_none() {
+                                panic = Some(p);
+                            }
+                        }
                     )*
-                    $fblock;
-                })
-            };
-            ($fblock:tt, $($blocks:tt),*) => {
-                // Reverse the order of the later blocks since Rayon executes them in reverse order
-                // when using a single thread. This ensures the execution order matches that
-                // of a single threaded rustc
-                parallel!(impl $fblock [] [$($blocks),*]);
+                    if let Some(panic) = panic {
+                        ::std::panic::resume_unwind(panic);
+                    }
+                }
             };
         }
 
-        pub use rayon::iter::ParallelIterator;
-        use rayon::iter::IntoParallelIterator;
+        use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
 
-        pub fn par_iter<T: IntoParallelIterator>(t: T) -> T::Iter {
-            t.into_par_iter()
+        pub fn par_for_each_in<I, T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>>(
+            t: T,
+            for_each: impl Fn(I) + DynSync + DynSend
+        ) {
+            if mode::active() {
+                let for_each = FromDyn::from(for_each);
+                let panic: Lock<Option<_>> = Lock::new(None);
+                t.into_par_iter().for_each(|i| if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) {
+                    let mut l = panic.lock();
+                    if l.is_none() {
+                        *l = Some(p)
+                    }
+                });
+
+                if let Some(panic) = panic.into_inner() {
+                    resume_unwind(panic);
+                }
+            } else {
+                // We catch panics here ensuring that all the loop iterations execute.
+                // This makes behavior consistent with the parallel compiler.
+                let mut panic = None;
+                t.into_iter().for_each(|i| {
+                    if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) {
+                        if panic.is_none() {
+                            panic = Some(p);
+                        }
+                    }
+                });
+                if let Some(panic) = panic {
+                    resume_unwind(panic);
+                }
+            }
         }
 
-        pub fn par_for_each_in<T: IntoParallelIterator>(
+        pub fn par_map<
+            I,
+            T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>,
+            R: std::marker::Send,
+            C: FromIterator<R> + FromParallelIterator<R>
+        >(
             t: T,
-            for_each: impl Fn(T::Item) + Sync + Send,
-        ) {
-            let ps: Vec<_> = t.into_par_iter().map(|i| catch_unwind(AssertUnwindSafe(|| for_each(i)))).collect();
-            ps.into_iter().for_each(|p| if let Err(panic) = p {
-                resume_unwind(panic)
-            });
+            map: impl Fn(I) -> R + DynSync + DynSend
+        ) -> C {
+            if mode::active() {
+                let panic: Lock<Option<_>> = Lock::new(None);
+                let map = FromDyn::from(map);
+                // We catch panics here ensuring that all the loop iterations execute.
+                let r = t.into_par_iter().filter_map(|i| {
+                    match catch_unwind(AssertUnwindSafe(|| map(i))) {
+                        Ok(r) => Some(r),
+                        Err(p) => {
+                            let mut l = panic.lock();
+                            if l.is_none() {
+                                *l = Some(p);
+                            }
+                            None
+                        },
+                    }
+                }).collect();
+
+                if let Some(panic) = panic.into_inner() {
+                    resume_unwind(panic);
+                }
+                r
+            } else {
+                // We catch panics here ensuring that all the loop iterations execute.
+                let mut panic = None;
+                let r = t.into_iter().filter_map(|i| {
+                    match catch_unwind(AssertUnwindSafe(|| map(i))) {
+                        Ok(r) => Some(r),
+                        Err(p) => {
+                            if panic.is_none() {
+                                panic = Some(p);
+                            }
+                            None
+                        }
+                    }
+                }).collect();
+                if let Some(panic) = panic {
+                    resume_unwind(panic);
+                }
+                r
+            }
         }
 
         pub type MetadataRef = OwnedSlice;
@@ -352,11 +514,6 @@ cfg_if! {
     }
 }
 
-pub fn assert_sync<T: ?Sized + Sync>() {}
-pub fn assert_send<T: ?Sized + Send>() {}
-pub fn assert_send_val<T: ?Sized + Send>(_t: &T) {}
-pub fn assert_send_sync_val<T: ?Sized + Sync + Send>(_t: &T) {}
-
 #[derive(Default)]
 #[cfg_attr(parallel_compiler, repr(align(64)))]
 pub struct CacheAligned<T>(pub T);
-- 
cgit 1.4.1-3-g733a5


From 261b727d7626506f0eabb88e99c9d6a27b2a4f24 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Mon, 13 Mar 2023 10:04:56 +0800
Subject: fix some nits

---
 compiler/rustc_data_structures/src/marker.rs | 41 ++++++++++------------------
 compiler/rustc_data_structures/src/sync.rs   | 25 ++++++++---------
 2 files changed, 27 insertions(+), 39 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index fef8177214b..2af7bc917c2 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -8,31 +8,20 @@ cfg_if!(
     } else {
         #[rustc_on_unimplemented(
             message = "`{Self}` doesn't implement `DynSend`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`",
-            label = "`{Self}` doesn't implement `DynSend`. \
             Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`"
         )]
-        // Ensure data structures is `Send` if `sync::active()` is true.
-        // `sync::active()` should be checked before using these data structures.
-        // Note: Ensure that the data structure **will not break**
-        // thread safety after being created.
-        //
-        // `sync::active()` should be checked when downcasting these data structures
-        // to `Send` via `FromDyn`.
+        // This is an auto trait for types which can be sent across threads if `sync::active()`
+        // is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
+        // `Send` type in `IntoDyn` will create a `DynSend` type.
         pub unsafe auto trait DynSend {}
 
         #[rustc_on_unimplemented(
             message = "`{Self}` doesn't implement `DynSync`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`",
-            label = "`{Self}` doesn't implement `DynSync`. \
             Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`"
         )]
-        // Ensure data structures is `Sync` if `sync::active()` is true.
-        // Note: Ensure that the data structure **will not break**
-        // thread safety after being checked.
-        //
-        // `sync::active()` should be checked when downcasting these data structures
-        // to `Send` via `FromDyn`.
+        // This is an auto trait for types which can be shared across threads if `sync::active()`
+        // is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
+        // `Sync` type in `IntoDyn` will create a `DynSync` type.
         pub unsafe auto trait DynSync {}
 
         // Same with `Sync` and `Send`.
@@ -110,8 +99,8 @@ cfg_if!(
             [thin_vec::ThinVec<T> where T: DynSend]
             [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
 
-            // We use `Send` here to omit some extra code, since they are only
-            // used in `Send` situations now.
+            // We use `Send` here, since they are only used in `Send` situations now.
+            // In this case we don't need copy or change the codes in `crate::owning_ref`.
             [crate::owning_ref::OwningRef<O, T> where O: Send, T: ?Sized + Send]
             [crate::owning_ref::OwningRefMut<O, T> where O: Send, T: ?Sized + Send]
         );
@@ -196,8 +185,8 @@ cfg_if!(
             [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
             [thin_vec::ThinVec<T> where T: DynSync]
 
-            // We use `Sync` here to omit some extra code, since they are only
-            // used in `Sync` situations now.
+            // We use `Sync` here, since they are only used in `Sync` situations now.
+            // In this case we don't need copy or change the codes in `crate::owning_ref`.
             [crate::owning_ref::OwningRef<O, T> where O: Sync, T: ?Sized + Sync]
             [crate::owning_ref::OwningRefMut<O, T> where O: Sync, T: ?Sized + Sync]
         );
@@ -213,11 +202,11 @@ pub fn assert_dyn_send_sync_val<T: ?Sized + DynSync + DynSend>(_t: &T) {}
 pub struct FromDyn<T>(T);
 
 impl<T> FromDyn<T> {
-    // Check `sync::active()` when creating this structure
-    // and downcasting to `Send`. So we can ensure it is
-    // thread-safe.
     #[inline(always)]
     pub fn from(val: T) -> Self {
+        // Check that `sync::active()` is true on creation so we can
+        // implement `Send` and `Sync` for this structure when `T`
+        // implements `DynSend` and `DynSync` respectively.
         #[cfg(parallel_compiler)]
         assert!(crate::sync::active());
         FromDyn(val)
@@ -229,11 +218,11 @@ impl<T> FromDyn<T> {
     }
 }
 
-// `FromDyn` is `Send` if `T` is `DynSend`, since it check when created.
+// `FromDyn` is `Send` if `T` is `DynSend`, since it ensures that sync::active() is true.
 #[cfg(parallel_compiler)]
 unsafe impl<T: DynSend> Send for FromDyn<T> {}
 
-// `FromDyn` is `Sync` if `T` is `DynSync`, since it check when created.
+// `FromDyn` is `Sync` if `T` is `DynSync`, since it ensures that sync::active() is true.
 #[cfg(parallel_compiler)]
 unsafe impl<T: DynSync> Sync for FromDyn<T> {}
 
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index 5f865486880..bffb98630c8 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -181,7 +181,7 @@ cfg_if! {
 
         #[macro_export]
         macro_rules! parallel {
-            ($($blocks:block),*) => {{
+            ($($blocks:block),*) => {
                 // We catch panics here ensuring that all the blocks execute.
                 // This makes behavior consistent with the parallel compiler.
                 let mut panic = None;
@@ -197,7 +197,7 @@ cfg_if! {
                 if let Some(panic) = panic {
                     ::std::panic::resume_unwind(panic);
                 }
-            }}
+            }
         }
 
         pub fn par_for_each_in<T: IntoIterator>(t: T, mut for_each: impl FnMut(T::Item) + Sync + Send) {
@@ -368,6 +368,7 @@ cfg_if! {
             }
         }
 
+        // This function only works when `mode::active()`.
         pub fn scope<'scope, OP, R>(op: OP) -> R
         where
             OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
@@ -381,24 +382,22 @@ cfg_if! {
         /// the current thread. Use that for the longest running block.
         #[macro_export]
         macro_rules! parallel {
-            ($fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => {
-                parallel!($fblock [$block, $($c,)*] [$($rest),*])
+            (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => {
+                parallel!(impl $fblock [$block, $($c,)*] [$($rest),*])
             };
-            ($fblock:block [$($blocks:expr,)*] []) => {
-                {
-                    ::rustc_data_structures::sync::scope(|s| {
-                        $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks);
-                        s.spawn(move |_| block.into_inner()());)*
-                        (|| $fblock)();
-                    });
-                }
+            (impl $fblock:block [$($blocks:expr,)*] []) => {
+                ::rustc_data_structures::sync::scope(|s| {
+                    $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks);
+                    s.spawn(move |_| block.into_inner()());)*
+                    (|| $fblock)();
+                });
             };
             ($fblock:block, $($blocks:block),*) => {
                 if rustc_data_structures::sync::active() {
                     // Reverse the order of the later blocks since Rayon executes them in reverse order
                     // when using a single thread. This ensures the execution order matches that
                     // of a single threaded rustc
-                    parallel!($fblock [] [$($blocks),*]);
+                    parallel!(impl $fblock [] [$($blocks),*]);
                 } else {
                     // We catch panics here ensuring that all the blocks execute.
                     // This makes behavior consistent with the parallel compiler.
-- 
cgit 1.4.1-3-g733a5


From f196e27d87cf2be019098c7562b4c2cf26566680 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Tue, 14 Mar 2023 20:31:58 +0800
Subject: fix `parallel!`

---
 compiler/rustc_data_structures/src/sync.rs | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index bffb98630c8..f30235572d0 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -402,6 +402,13 @@ cfg_if! {
                     // We catch panics here ensuring that all the blocks execute.
                     // This makes behavior consistent with the parallel compiler.
                     let mut panic = None;
+                    if let Err(p) = ::std::panic::catch_unwind(
+                        ::std::panic::AssertUnwindSafe(|| $fblock)
+                    ) {
+                        if panic.is_none() {
+                            panic = Some(p);
+                        }
+                    }
                     $(
                         if let Err(p) = ::std::panic::catch_unwind(
                             ::std::panic::AssertUnwindSafe(|| $blocks)
-- 
cgit 1.4.1-3-g733a5


From 9f8ab2a8d361c537d5f0e3c27df9e3f630daecd3 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Tue, 4 Apr 2023 16:26:00 +0800
Subject: rename relative names in `sync`

---
 compiler/rustc_data_structures/src/marker.rs | 12 ++++-----
 compiler/rustc_data_structures/src/sync.rs   | 39 ++++++++++++++++------------
 compiler/rustc_interface/src/interface.rs    |  2 +-
 3 files changed, 29 insertions(+), 24 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index 2af7bc917c2..62e65209d72 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -10,7 +10,7 @@ cfg_if!(
             message = "`{Self}` doesn't implement `DynSend`. \
             Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`"
         )]
-        // This is an auto trait for types which can be sent across threads if `sync::active()`
+        // This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
         // is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
         // `Send` type in `IntoDyn` will create a `DynSend` type.
         pub unsafe auto trait DynSend {}
@@ -19,7 +19,7 @@ cfg_if!(
             message = "`{Self}` doesn't implement `DynSync`. \
             Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`"
         )]
-        // This is an auto trait for types which can be shared across threads if `sync::active()`
+        // This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
         // is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
         // `Sync` type in `IntoDyn` will create a `DynSync` type.
         pub unsafe auto trait DynSync {}
@@ -204,11 +204,11 @@ pub struct FromDyn<T>(T);
 impl<T> FromDyn<T> {
     #[inline(always)]
     pub fn from(val: T) -> Self {
-        // Check that `sync::active()` is true on creation so we can
+        // Check that `sync::is_dyn_thread_safe()` is true on creation so we can
         // implement `Send` and `Sync` for this structure when `T`
         // implements `DynSend` and `DynSync` respectively.
         #[cfg(parallel_compiler)]
-        assert!(crate::sync::active());
+        assert!(crate::sync::is_dyn_thread_safe());
         FromDyn(val)
     }
 
@@ -218,11 +218,11 @@ impl<T> FromDyn<T> {
     }
 }
 
-// `FromDyn` is `Send` if `T` is `DynSend`, since it ensures that sync::active() is true.
+// `FromDyn` is `Send` if `T` is `DynSend`, since it ensures that sync::is_dyn_thread_safe() is true.
 #[cfg(parallel_compiler)]
 unsafe impl<T: DynSend> Send for FromDyn<T> {}
 
-// `FromDyn` is `Sync` if `T` is `DynSync`, since it ensures that sync::active() is true.
+// `FromDyn` is `Sync` if `T` is `DynSync`, since it ensures that sync::is_dyn_thread_safe() is true.
 #[cfg(parallel_compiler)]
 unsafe impl<T: DynSync> Sync for FromDyn<T> {}
 
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index f30235572d0..41aad2ff4e4 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -61,32 +61,37 @@ mod mode {
     use std::sync::atomic::AtomicU8;
 
     const UNINITIALIZED: u8 = 0;
-    const INACTIVE: u8 = 1;
-    const ACTIVE: u8 = 2;
+    const DYN_NOT_SYNC: u8 = 1;
+    const DYN_SYNC: u8 = 2;
 
-    static MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED);
+    static DYN_SYNC_MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED);
 
+    // Weather control thread safety dynamically
     #[inline]
-    pub fn active() -> bool {
-        match MODE.load(Ordering::Relaxed) {
-            INACTIVE => false,
-            ACTIVE => true,
+    pub fn is_dyn_thread_safe() -> bool {
+        match DYN_SYNC_MODE.load(Ordering::Relaxed) {
+            DYN_NOT_SYNC => false,
+            DYN_SYNC => true,
             _ => panic!("uninitialized parallel mode!"),
         }
     }
 
     // Only set by the `-Z threads` compile option
-    pub fn set(parallel: bool) {
-        let set: u8 = if parallel { ACTIVE } else { INACTIVE };
-        let previous =
-            MODE.compare_exchange(UNINITIALIZED, set, Ordering::Relaxed, Ordering::Relaxed);
+    pub fn set_dyn_thread_safe_mode(parallel: bool) {
+        let set: u8 = if parallel { DYN_SYNC } else { DYN_NOT_SYNC };
+        let previous = DYN_SYNC_MODE.compare_exchange(
+            UNINITIALIZED,
+            set,
+            Ordering::Relaxed,
+            Ordering::Relaxed,
+        );
 
         // Check that the mode was either uninitialized or was already set to the requested mode.
         assert!(previous.is_ok() || previous == Err(set));
     }
 }
 
-pub use mode::{active, set};
+pub use mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode};
 cfg_if! {
     if #[cfg(not(parallel_compiler))] {
         pub unsafe auto trait Send {}
@@ -358,7 +363,7 @@ cfg_if! {
             A: FnOnce() -> RA + DynSend,
             B: FnOnce() -> RB + DynSend,
         {
-            if mode::active() {
+            if mode::is_dyn_thread_safe() {
                 let oper_a = FromDyn::from(oper_a);
                 let oper_b = FromDyn::from(oper_b);
                 let (a, b) = rayon::join(move || FromDyn::from(oper_a.into_inner()()), move || FromDyn::from(oper_b.into_inner()()));
@@ -368,7 +373,7 @@ cfg_if! {
             }
         }
 
-        // This function only works when `mode::active()`.
+        // This function only works when `mode::is_dyn_thread_safe()`.
         pub fn scope<'scope, OP, R>(op: OP) -> R
         where
             OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
@@ -393,7 +398,7 @@ cfg_if! {
                 });
             };
             ($fblock:block, $($blocks:block),*) => {
-                if rustc_data_structures::sync::active() {
+                if rustc_data_structures::sync::is_dyn_thread_safe() {
                     // Reverse the order of the later blocks since Rayon executes them in reverse order
                     // when using a single thread. This ensures the execution order matches that
                     // of a single threaded rustc
@@ -431,7 +436,7 @@ cfg_if! {
             t: T,
             for_each: impl Fn(I) + DynSync + DynSend
         ) {
-            if mode::active() {
+            if mode::is_dyn_thread_safe() {
                 let for_each = FromDyn::from(for_each);
                 let panic: Lock<Option<_>> = Lock::new(None);
                 t.into_par_iter().for_each(|i| if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) {
@@ -470,7 +475,7 @@ cfg_if! {
             t: T,
             map: impl Fn(I) -> R + DynSync + DynSend
         ) -> C {
-            if mode::active() {
+            if mode::is_dyn_thread_safe() {
                 let panic: Lock<Option<_>> = Lock::new(None);
                 let map = FromDyn::from(map);
                 // We catch panics here ensuring that all the loop iterations execute.
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 2a091077700..fb6c273e929 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -62,7 +62,7 @@ impl Compiler {
 
 #[allow(rustc::bad_opt_access)]
 pub fn set_parallel_mode(sopts: &config::UnstableOptions) {
-    rustc_data_structures::sync::set(sopts.threads > 1);
+    rustc_data_structures::sync::set_dyn_thread_safe_mode(sopts.threads > 1);
 }
 
 /// Converts strings provided as `--cfg [cfgspec]` into a `crate_cfg`.
-- 
cgit 1.4.1-3-g733a5


From 089a38880b6e3b05a2c59644469233dbda411158 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Fri, 7 Apr 2023 21:20:26 +0800
Subject: correct literals for dyn thread safe

---
 compiler/rustc_ast/src/tokenstream.rs        |  2 +-
 compiler/rustc_data_structures/src/marker.rs | 21 ++++++++++++---------
 compiler/rustc_data_structures/src/sync.rs   | 24 ++++++++++++------------
 compiler/rustc_driver_impl/src/lib.rs        |  4 ++--
 compiler/rustc_error_messages/src/lib.rs     |  8 ++++----
 compiler/rustc_errors/src/lib.rs             |  6 +++---
 compiler/rustc_errors/src/tests.rs           |  8 +++++---
 compiler/rustc_interface/src/interface.rs    |  2 +-
 compiler/rustc_middle/src/ty/list.rs         |  2 +-
 compiler/rustc_span/src/source_map.rs        |  8 +++++---
 src/librustdoc/lib.rs                        |  4 ++--
 11 files changed, 48 insertions(+), 41 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 3b47ebef2fd..3f0b1627afa 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -48,7 +48,7 @@ pub enum TokenTree {
     Delimited(DelimSpan, Delimiter, TokenStream),
 }
 
-// Ensure all fields of `TokenTree` is `DynSend` and `DynSync`.
+// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
 #[cfg(parallel_compiler)]
 fn _dummy()
 where
diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index 62e65209d72..6dad2bac58a 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -8,20 +8,20 @@ cfg_if!(
     } else {
         #[rustc_on_unimplemented(
             message = "`{Self}` doesn't implement `DynSend`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Send`"
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Send`"
         )]
         // This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
         // is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
-        // `Send` type in `IntoDyn` will create a `DynSend` type.
+        // `Send` type in `IntoDynSyncSend` will create a `DynSend` type.
         pub unsafe auto trait DynSend {}
 
         #[rustc_on_unimplemented(
             message = "`{Self}` doesn't implement `DynSync`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDyn` if it's already `Sync`"
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Sync`"
         )]
         // This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
         // is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
-        // `Sync` type in `IntoDyn` will create a `DynSync` type.
+        // `Sync` type in `IntoDynSyncSend` will create a `DynSync` type.
         pub unsafe auto trait DynSync {}
 
         // Same with `Sync` and `Send`.
@@ -234,15 +234,18 @@ impl<T> const std::ops::Deref for FromDyn<T> {
     }
 }
 
+// A wrapper to convert a struct that is already a `Send` or `Sync` into
+// an instance of `DynSend` and `DynSync`, since the compiler cannot infer
+// it automatically in some cases. (e.g. Box<dyn Send / Sync>)
 #[derive(Copy, Clone)]
-pub struct IntoDyn<T: ?Sized>(pub T);
+pub struct IntoDynSyncSend<T: ?Sized>(pub T);
 
 #[cfg(parallel_compiler)]
-unsafe impl<T: ?Sized + Send> DynSend for IntoDyn<T> {}
+unsafe impl<T: ?Sized + Send> DynSend for IntoDynSyncSend<T> {}
 #[cfg(parallel_compiler)]
-unsafe impl<T: ?Sized + Sync> DynSync for IntoDyn<T> {}
+unsafe impl<T: ?Sized + Sync> DynSync for IntoDynSyncSend<T> {}
 
-impl<T> const std::ops::Deref for IntoDyn<T> {
+impl<T> const std::ops::Deref for IntoDynSyncSend<T> {
     type Target = T;
 
     fn deref(&self) -> &T {
@@ -250,7 +253,7 @@ impl<T> const std::ops::Deref for IntoDyn<T> {
     }
 }
 
-impl<T> const std::ops::DerefMut for IntoDyn<T> {
+impl<T> const std::ops::DerefMut for IntoDynSyncSend<T> {
     fn deref_mut(&mut self) -> &mut T {
         &mut self.0
     }
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index 41aad2ff4e4..8a778866a77 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -61,25 +61,25 @@ mod mode {
     use std::sync::atomic::AtomicU8;
 
     const UNINITIALIZED: u8 = 0;
-    const DYN_NOT_SYNC: u8 = 1;
-    const DYN_SYNC: u8 = 2;
+    const DYN_NOT_THREAD_SAFE: u8 = 1;
+    const DYN_THREAD_SAFE: u8 = 2;
 
-    static DYN_SYNC_MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED);
+    static DYN_THREAD_SAFE_MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED);
 
-    // Weather control thread safety dynamically
+    // Whether thread safety is enabled (due to running under multiple threads).
     #[inline]
     pub fn is_dyn_thread_safe() -> bool {
-        match DYN_SYNC_MODE.load(Ordering::Relaxed) {
-            DYN_NOT_SYNC => false,
-            DYN_SYNC => true,
-            _ => panic!("uninitialized parallel mode!"),
+        match DYN_THREAD_SAFE_MODE.load(Ordering::Relaxed) {
+            DYN_NOT_THREAD_SAFE => false,
+            DYN_THREAD_SAFE => true,
+            _ => panic!("uninitialized dyn_thread_safe mode!"),
         }
     }
 
     // Only set by the `-Z threads` compile option
-    pub fn set_dyn_thread_safe_mode(parallel: bool) {
-        let set: u8 = if parallel { DYN_SYNC } else { DYN_NOT_SYNC };
-        let previous = DYN_SYNC_MODE.compare_exchange(
+    pub fn set_dyn_thread_safe_mode(mode: bool) {
+        let set: u8 = if mode { DYN_THREAD_SAFE } else { DYN_NOT_THREAD_SAFE };
+        let previous = DYN_THREAD_SAFE_MODE.compare_exchange(
             UNINITIALIZED,
             set,
             Ordering::Relaxed,
@@ -401,7 +401,7 @@ cfg_if! {
                 if rustc_data_structures::sync::is_dyn_thread_safe() {
                     // Reverse the order of the later blocks since Rayon executes them in reverse order
                     // when using a single thread. This ensures the execution order matches that
-                    // of a single threaded rustc
+                    // of a single threaded rustc.
                     parallel!(impl $fblock [] [$($blocks),*]);
                 } else {
                     // We catch panics here ensuring that all the blocks execute.
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 353f49e6619..b94fb335a4e 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -255,8 +255,8 @@ fn run_compiler(
 
     let sopts = config::build_session_options(&matches);
 
-    // Set parallel mode before thread pool creation as the session will already create locks.
-    interface::set_parallel_mode(&sopts.unstable_opts);
+    // Set parallel mode before thread pool creation, which will create `Lock`s.
+    interface::set_thread_safe_mode(&sopts.unstable_opts);
 
     if let Some(ref code) = matches.opt_str("explain") {
         handle_explain(diagnostics_registry(), code, sopts.error_format);
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 2b7c8dc5f3d..0accb4ab96f 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -11,7 +11,7 @@ extern crate tracing;
 use fluent_bundle::FluentResource;
 use fluent_syntax::parser::ParserError;
 use icu_provider_adapters::fallback::{LocaleFallbackProvider, LocaleFallbacker};
-use rustc_data_structures::sync::{IntoDyn, Lrc};
+use rustc_data_structures::sync::{IntoDynSyncSend, Lrc};
 use rustc_fluent_macro::fluent_messages;
 use rustc_macros::{Decodable, Encodable};
 use rustc_span::Span;
@@ -38,16 +38,16 @@ pub use unic_langid::{langid, LanguageIdentifier};
 fluent_messages! { "../messages.ftl" }
 
 pub type FluentBundle =
-    IntoDyn<fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>>;
+    IntoDynSyncSend<fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>>;
 
 #[cfg(not(parallel_compiler))]
 fn new_bundle(locales: Vec<LanguageIdentifier>) -> FluentBundle {
-    IntoDyn(fluent_bundle::bundle::FluentBundle::new(locales))
+    IntoDynSyncSend(fluent_bundle::bundle::FluentBundle::new(locales))
 }
 
 #[cfg(parallel_compiler)]
 fn new_bundle(locales: Vec<LanguageIdentifier>) -> FluentBundle {
-    IntoDyn(fluent_bundle::bundle::FluentBundle::new_concurrent(locales))
+    IntoDynSyncSend(fluent_bundle::bundle::FluentBundle::new_concurrent(locales))
 }
 
 #[derive(Debug)]
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index f57bb8d8bdc..64fbf22c262 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -32,7 +32,7 @@ use emitter::{is_case_difference, Emitter, EmitterWriter};
 use registry::Registry;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::{Hash128, StableHasher};
-use rustc_data_structures::sync::{self, IntoDyn, Lock, Lrc};
+use rustc_data_structures::sync::{self, IntoDynSyncSend, Lock, Lrc};
 use rustc_data_structures::AtomicRef;
 pub use rustc_error_messages::{
     fallback_fluent_bundle, fluent_bundle, DelayDm, DiagnosticMessage, FluentBundle,
@@ -409,7 +409,7 @@ struct HandlerInner {
     err_count: usize,
     warn_count: usize,
     deduplicated_err_count: usize,
-    emitter: IntoDyn<Box<dyn Emitter + sync::Send>>,
+    emitter: IntoDynSyncSend<Box<dyn Emitter + sync::Send>>,
     delayed_span_bugs: Vec<DelayedDiagnostic>,
     delayed_good_path_bugs: Vec<DelayedDiagnostic>,
     /// This flag indicates that an expected diagnostic was emitted and suppressed.
@@ -605,7 +605,7 @@ impl Handler {
                 warn_count: 0,
                 deduplicated_err_count: 0,
                 deduplicated_warn_count: 0,
-                emitter: IntoDyn(emitter),
+                emitter: IntoDynSyncSend(emitter),
                 delayed_span_bugs: Vec::new(),
                 delayed_good_path_bugs: Vec::new(),
                 suppressed_expected_diag: false,
diff --git a/compiler/rustc_errors/src/tests.rs b/compiler/rustc_errors/src/tests.rs
index 08b3281d9a5..0e729b71680 100644
--- a/compiler/rustc_errors/src/tests.rs
+++ b/compiler/rustc_errors/src/tests.rs
@@ -2,7 +2,7 @@ use crate::error::{TranslateError, TranslateErrorKind};
 use crate::fluent_bundle::*;
 use crate::translation::Translate;
 use crate::FluentBundle;
-use rustc_data_structures::sync::{IntoDyn, Lrc};
+use rustc_data_structures::sync::{IntoDynSyncSend, Lrc};
 use rustc_error_messages::fluent_bundle::resolver::errors::{ReferenceKind, ResolverError};
 use rustc_error_messages::langid;
 use rustc_error_messages::DiagnosticMessage;
@@ -28,11 +28,13 @@ fn make_dummy(ftl: &'static str) -> Dummy {
 
     #[cfg(parallel_compiler)]
     let mut bundle: FluentBundle =
-        IntoDyn(crate::fluent_bundle::bundle::FluentBundle::new_concurrent(vec![langid_en]));
+        IntoDynSyncSend(crate::fluent_bundle::bundle::FluentBundle::new_concurrent(vec![
+            langid_en,
+        ]));
 
     #[cfg(not(parallel_compiler))]
     let mut bundle: FluentBundle =
-        IntoDyn(crate::fluent_bundle::bundle::FluentBundle::new(vec![langid_en]));
+        IntoDynSyncSend(crate::fluent_bundle::bundle::FluentBundle::new(vec![langid_en]));
 
     bundle.add_resource(resource).expect("Failed to add FTL resources to the bundle.");
 
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index fb6c273e929..ee373f72ee9 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -61,7 +61,7 @@ impl Compiler {
 }
 
 #[allow(rustc::bad_opt_access)]
-pub fn set_parallel_mode(sopts: &config::UnstableOptions) {
+pub fn set_thread_safe_mode(sopts: &config::UnstableOptions) {
     rustc_data_structures::sync::set_dyn_thread_safe_mode(sopts.threads > 1);
 }
 
diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs
index 8f99a051328..71911a5a618 100644
--- a/compiler/rustc_middle/src/ty/list.rs
+++ b/compiler/rustc_middle/src/ty/list.rs
@@ -199,7 +199,7 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
 
 unsafe impl<T: Sync> Sync for List<T> {}
 
-// We need this since `List` uses extern type `OpaqueListContents`
+// We need this since `List` uses extern type `OpaqueListContents`.
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sync::DynSync;
 #[cfg(parallel_compiler)]
diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs
index e0a2733a464..8238a16969d 100644
--- a/compiler/rustc_span/src/source_map.rs
+++ b/compiler/rustc_span/src/source_map.rs
@@ -14,7 +14,9 @@ pub use crate::*;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{Hash128, Hash64, StableHasher};
-use rustc_data_structures::sync::{AtomicU32, IntoDyn, Lrc, MappedReadGuard, ReadGuard, RwLock};
+use rustc_data_structures::sync::{
+    AtomicU32, IntoDynSyncSend, Lrc, MappedReadGuard, ReadGuard, RwLock,
+};
 use std::cmp;
 use std::hash::Hash;
 use std::path::{self, Path, PathBuf};
@@ -176,7 +178,7 @@ pub struct SourceMap {
     used_address_space: AtomicU32,
 
     files: RwLock<SourceMapFiles>,
-    file_loader: IntoDyn<Box<dyn FileLoader + Sync + Send>>,
+    file_loader: IntoDynSyncSend<Box<dyn FileLoader + Sync + Send>>,
     // This is used to apply the file path remapping as specified via
     // `--remap-path-prefix` to all `SourceFile`s allocated within this `SourceMap`.
     path_mapping: FilePathMapping,
@@ -202,7 +204,7 @@ impl SourceMap {
         SourceMap {
             used_address_space: AtomicU32::new(0),
             files: Default::default(),
-            file_loader: IntoDyn(file_loader),
+            file_loader: IntoDynSyncSend(file_loader),
             path_mapping,
             hash_kind,
         }
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index ba71d0372a7..70b6ba1aced 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -730,8 +730,8 @@ fn main_args(at_args: &[String]) -> MainResult {
         }
     };
 
-    // Set parallel mode early as the error handler will already create locks.
-    interface::set_parallel_mode(&options.unstable_opts);
+    // Set parallel mode before error handler creation, which will create `Lock`s.
+    interface::set_thread_safe_mode(&options.unstable_opts);
 
     let diag = core::new_handler(
         options.error_format,
-- 
cgit 1.4.1-3-g733a5


From bffccddac3470f68efe15a3e81257869be562568 Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Mon, 10 Apr 2023 09:53:50 +0800
Subject: correct import of owned_slice

---
 compiler/rustc_data_structures/src/lib.rs               |  3 ++-
 compiler/rustc_data_structures/src/marker.rs            | 12 ++----------
 compiler/rustc_data_structures/src/owned_slice/tests.rs |  4 ++--
 3 files changed, 6 insertions(+), 13 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index f543e0d4a7a..579f54c5f71 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -29,7 +29,8 @@
 #![feature(allocator_api)]
 #![feature(get_mut_unchecked)]
 #![feature(lint_reasons)]
-#![feature(unwrap_infallible)]#![feature(const_mut_refs)]
+#![feature(unwrap_infallible)]
+#![feature(const_mut_refs)]
 #![feature(const_trait_impl)]
 #![feature(strict_provenance)]
 #![feature(ptr_alignment_type)]
diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index 6dad2bac58a..e23a10839ee 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -72,6 +72,7 @@ cfg_if!(
             [rustc_arena::DroplessArena]
             [crate::memmap::Mmap]
             [crate::profiling::SelfProfiler]
+            [crate::owned_slice::OwnedSlice]
         );
 
         macro_rules! impl_dyn_send {
@@ -98,11 +99,6 @@ cfg_if!(
             [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
             [thin_vec::ThinVec<T> where T: DynSend]
             [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
-
-            // We use `Send` here, since they are only used in `Send` situations now.
-            // In this case we don't need copy or change the codes in `crate::owning_ref`.
-            [crate::owning_ref::OwningRef<O, T> where O: Send, T: ?Sized + Send]
-            [crate::owning_ref::OwningRefMut<O, T> where O: Send, T: ?Sized + Send]
         );
 
         macro_rules! impls_dyn_sync_neg {
@@ -154,6 +150,7 @@ cfg_if!(
             [jobserver_crate::Client]
             [crate::memmap::Mmap]
             [crate::profiling::SelfProfiler]
+            [crate::owned_slice::OwnedSlice]
         );
 
         macro_rules! impl_dyn_sync {
@@ -184,11 +181,6 @@ cfg_if!(
             [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
             [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
             [thin_vec::ThinVec<T> where T: DynSync]
-
-            // We use `Sync` here, since they are only used in `Sync` situations now.
-            // In this case we don't need copy or change the codes in `crate::owning_ref`.
-            [crate::owning_ref::OwningRef<O, T> where O: Sync, T: ?Sized + Sync]
-            [crate::owning_ref::OwningRefMut<O, T> where O: Sync, T: ?Sized + Sync]
         );
     }
 );
diff --git a/compiler/rustc_data_structures/src/owned_slice/tests.rs b/compiler/rustc_data_structures/src/owned_slice/tests.rs
index e715fb55362..e151b8c2de0 100644
--- a/compiler/rustc_data_structures/src/owned_slice/tests.rs
+++ b/compiler/rustc_data_structures/src/owned_slice/tests.rs
@@ -69,6 +69,6 @@ fn drop_drops() {
 
 #[test]
 fn send_sync() {
-    crate::sync::assert_send::<OwnedSlice>();
-    crate::sync::assert_sync::<OwnedSlice>();
+    crate::sync::assert_dyn_send::<OwnedSlice>();
+    crate::sync::assert_dyn_sync::<OwnedSlice>();
 }
-- 
cgit 1.4.1-3-g733a5


From d7e3e5bede187d113fa01c4d8b8c16a2bd4f721c Mon Sep 17 00:00:00 2001
From: SparrowLii <liyuan179@huawei.com>
Date: Sat, 6 May 2023 10:23:51 +0800
Subject: add `DynSend / DynSync` for `CopyTaggedPtr`

---
 compiler/rustc_data_structures/src/lib.rs    |  2 --
 compiler/rustc_data_structures/src/marker.rs | 11 ++++++++---
 compiler/rustc_middle/src/ty/query.rs        |  5 +++--
 3 files changed, 11 insertions(+), 7 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 579f54c5f71..5b9b0e106d2 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -30,8 +30,6 @@
 #![feature(get_mut_unchecked)]
 #![feature(lint_reasons)]
 #![feature(unwrap_infallible)]
-#![feature(const_mut_refs)]
-#![feature(const_trait_impl)]
 #![feature(strict_provenance)]
 #![feature(ptr_alignment_type)]
 #![feature(macro_metavar_expr)]
diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index e23a10839ee..f8c06f9a814 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -94,6 +94,7 @@ cfg_if!(
             [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
             [crate::sync::Lock<T> where T: DynSend]
             [crate::sync::RwLock<T> where T: DynSend]
+            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
             [rustc_arena::TypedArena<T> where T: DynSend]
             [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
             [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
@@ -175,6 +176,7 @@ cfg_if!(
             [crate::sync::OneThread<T> where T]
             [crate::sync::WorkerLocal<T> where T: DynSend]
             [crate::intern::Interned<'a, T> where 'a, T: DynSync]
+            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
             [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
             [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
             [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
@@ -218,9 +220,10 @@ unsafe impl<T: DynSend> Send for FromDyn<T> {}
 #[cfg(parallel_compiler)]
 unsafe impl<T: DynSync> Sync for FromDyn<T> {}
 
-impl<T> const std::ops::Deref for FromDyn<T> {
+impl<T> std::ops::Deref for FromDyn<T> {
     type Target = T;
 
+    #[inline(always)]
     fn deref(&self) -> &Self::Target {
         &self.0
     }
@@ -237,15 +240,17 @@ unsafe impl<T: ?Sized + Send> DynSend for IntoDynSyncSend<T> {}
 #[cfg(parallel_compiler)]
 unsafe impl<T: ?Sized + Sync> DynSync for IntoDynSyncSend<T> {}
 
-impl<T> const std::ops::Deref for IntoDynSyncSend<T> {
+impl<T> std::ops::Deref for IntoDynSyncSend<T> {
     type Target = T;
 
+    #[inline(always)]
     fn deref(&self) -> &T {
         &self.0
     }
 }
 
-impl<T> const std::ops::DerefMut for IntoDynSyncSend<T> {
+impl<T> std::ops::DerefMut for IntoDynSyncSend<T> {
+    #[inline(always)]
     fn deref_mut(&mut self) -> &mut T {
         &mut self.0
     }
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 1d09a62ca17..07d47cae5ee 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -52,8 +52,9 @@ use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::AtomicU64;use rustc_data_structures::sync::WorkerLocal;
-use rustc_data_structures::sync::{self, Lrc};
+use rustc_data_structures::sync::AtomicU64;
+use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::WorkerLocal;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-- 
cgit 1.4.1-3-g733a5


From f16d2b1629d62db306bd9d03676524dc848c94b4 Mon Sep 17 00:00:00 2001
From: Tomasz Miąsko <tomasz.miasko@gmail.com>
Date: Sun, 14 May 2023 00:00:00 +0000
Subject: Start node has no immediate dominator

Change the immediate_dominator return type to Option, and use None to
indicate that node has no immediate dominator.

Also fix the issue where the start node would be returned as its own
immediate dominator.
---
 .../rustc_const_eval/src/transform/validate.rs     |  2 +-
 .../src/graph/dominators/mod.rs                    | 22 ++++++++++------------
 .../src/graph/dominators/tests.rs                  | 14 ++++++++++++--
 3 files changed, 23 insertions(+), 15 deletions(-)

(limited to 'compiler/rustc_data_structures/src')

diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index c50e937d84f..3c350e25ba6 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -164,7 +164,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 if let Some(root) = post_contract_node.get(&bb) {
                     break *root;
                 }
-                let parent = doms.immediate_dominator(bb);
+                let parent = doms.immediate_dominator(bb).unwrap();
                 dom_path.push(bb);
                 if !self.body.basic_blocks[parent].is_cleanup {
                     break bb;
diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
index e76bdac2864..a7de709ba72 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
@@ -242,7 +242,9 @@ pub fn dominators<G: ControlFlowGraph>(graph: G) -> Dominators<G::Node> {
         immediate_dominators[*node] = Some(pre_order_to_real[idom[idx]]);
     }
 
-    Dominators { post_order_rank, immediate_dominators }
+    let start_node = graph.start_node();
+    immediate_dominators[start_node] = None;
+    Dominators { start_node, post_order_rank, immediate_dominators }
 }
 
 /// Evaluate the link-eval virtual forest, providing the currently minimum semi
@@ -308,6 +310,7 @@ fn compress(
 /// Tracks the list of dominators for each node.
 #[derive(Clone, Debug)]
 pub struct Dominators<N: Idx> {
+    start_node: N,
     post_order_rank: IndexVec<N, usize>,
     // Even though we track only the immediate dominator of each node, it's
     // possible to get its full list of dominators by looking up the dominator
@@ -316,14 +319,14 @@ pub struct Dominators<N: Idx> {
 }
 
 impl<Node: Idx> Dominators<Node> {
-    /// Whether the given Node has an immediate dominator.
+    /// Returns true if node is reachable from the start node.
     pub fn is_reachable(&self, node: Node) -> bool {
-        self.immediate_dominators[node].is_some()
+        node == self.start_node || self.immediate_dominators[node].is_some()
     }
 
-    pub fn immediate_dominator(&self, node: Node) -> Node {
-        assert!(self.is_reachable(node), "node {node:?} is not reachable");
-        self.immediate_dominators[node].unwrap()
+    /// Returns the immediate dominator of node, if any.
+    pub fn immediate_dominator(&self, node: Node) -> Option<Node> {
+        self.immediate_dominators[node]
     }
 
     /// Provides an iterator over each dominator up the CFG, for the given Node.
@@ -357,12 +360,7 @@ impl<'dom, Node: Idx> Iterator for Iter<'dom, Node> {
 
     fn next(&mut self) -> Option<Self::Item> {
         if let Some(node) = self.node {
-            let dom = self.dominators.immediate_dominator(node);
-            if dom == node {
-                self.node = None; // reached the root
-            } else {
-                self.node = Some(dom);
-            }
+            self.node = self.dominators.immediate_dominator(node);
             Some(node)
         } else {
             None
diff --git a/compiler/rustc_data_structures/src/graph/dominators/tests.rs b/compiler/rustc_data_structures/src/graph/dominators/tests.rs
index ff31d8f7fdc..8b124516623 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/tests.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/tests.rs
@@ -8,7 +8,7 @@ fn diamond() {
 
     let dominators = dominators(&graph);
     let immediate_dominators = &dominators.immediate_dominators;
-    assert_eq!(immediate_dominators[0], Some(0));
+    assert_eq!(immediate_dominators[0], None);
     assert_eq!(immediate_dominators[1], Some(0));
     assert_eq!(immediate_dominators[2], Some(0));
     assert_eq!(immediate_dominators[3], Some(0));
@@ -30,7 +30,7 @@ fn paper() {
     assert_eq!(immediate_dominators[3], Some(6));
     assert_eq!(immediate_dominators[4], Some(6));
     assert_eq!(immediate_dominators[5], Some(6));
-    assert_eq!(immediate_dominators[6], Some(6));
+    assert_eq!(immediate_dominators[6], None);
 }
 
 #[test]
@@ -43,3 +43,13 @@ fn paper_slt() {
 
     dominators(&graph);
 }
+
+#[test]
+fn immediate_dominator() {
+    let graph = TestGraph::new(1, &[(1, 2), (2, 3)]);
+    let dominators = dominators(&graph);
+    assert_eq!(dominators.immediate_dominator(0), None);
+    assert_eq!(dominators.immediate_dominator(1), None);
+    assert_eq!(dominators.immediate_dominator(2), Some(1));
+    assert_eq!(dominators.immediate_dominator(3), Some(2));
+}
-- 
cgit 1.4.1-3-g733a5