Merge branch 'master' into hooks

16 files changed, 1052 insertions, 56 deletions

diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
index 438a0d0c6ff..1cfbce2355e 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
@@ -9,6 +9,7 @@ use super::iterate::reverse_post_order;
 use super::ControlFlowGraph;
 use rustc_index::vec::{Idx, IndexVec};
 use std::borrow::BorrowMut;
+use std::cmp::Ordering;
 
 #[cfg(test)]
 mod tests;
@@ -108,6 +109,14 @@ impl<Node: Idx> Dominators<Node> {
         // FIXME -- could be optimized by using post-order-rank
         self.dominators(node).any(|n| n == dom)
     }
+
+    /// Provide deterministic ordering of nodes such that, if any two nodes have a dominator
+    /// relationship, the dominator will always precede the dominated. (The relative ordering
+    /// of two unrelated nodes will also be consistent, but otherwise the order has no
+    /// meaning.) This method cannot be used to determine if either Node dominates the other.
+    pub fn rank_partial_cmp(&self, lhs: Node, rhs: Node) -> Option<Ordering> {
+        self.post_order_rank[lhs].partial_cmp(&self.post_order_rank[rhs])
+    }
 }
 
 pub struct Iter<'dom, Node: Idx> {
diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
index 64ff6130ddf..bc3d1ce53ba 100644
--- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
@@ -87,11 +87,8 @@ where
 }
 
 /// Allows searches to terminate early with a value.
-#[derive(Clone, Copy, Debug)]
-pub enum ControlFlow<T> {
-    Break(T),
-    Continue,
-}
+// FIXME (#75744): remove the alias once the generics are in a better order and `C=()`.
+pub type ControlFlow<T> = std::ops::ControlFlow<(), T>;
 
 /// The status of a node in the depth-first search.
 ///
@@ -260,12 +257,12 @@ where
         _node: G::Node,
         _prior_status: Option<NodeStatus>,
     ) -> ControlFlow<Self::BreakVal> {
-        ControlFlow::Continue
+        ControlFlow::CONTINUE
     }
 
     /// Called after all nodes reachable from this one have been examined.
     fn node_settled(&mut self, _node: G::Node) -> ControlFlow<Self::BreakVal> {
-        ControlFlow::Continue
+        ControlFlow::CONTINUE
     }
 
     /// Behave as if no edges exist from `source` to `target`.
@@ -289,8 +286,8 @@ where
         prior_status: Option<NodeStatus>,
     ) -> ControlFlow<Self::BreakVal> {
         match prior_status {
-            Some(NodeStatus::Visited) => ControlFlow::Break(()),
-            _ => ControlFlow::Continue,
+            Some(NodeStatus::Visited) => ControlFlow::BREAK,
+            _ => ControlFlow::CONTINUE,
         }
     }
 }
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index de4e7a13424..9958e5dd5e0 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -6,13 +6,14 @@
 //!
 //! This API is completely unstable and subject to change.
 
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/")]
-#![allow(incomplete_features)]
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(array_windows)]
+#![feature(control_flow_enum)]
 #![feature(in_band_lifetimes)]
 #![feature(unboxed_closures)]
-#![feature(generators)]
 #![feature(generator_trait)]
 #![feature(fn_traits)]
+#![feature(int_bits_const)]
 #![feature(min_specialization)]
 #![feature(optin_builtin_traits)]
 #![feature(nll)]
@@ -25,7 +26,7 @@
 #![feature(thread_id_value)]
 #![feature(extend_one)]
 #![feature(const_panic)]
-#![feature(const_generics)]
+#![feature(min_const_generics)]
 #![feature(once_cell)]
 #![allow(rustc::default_hash_types)]
 
@@ -86,25 +87,27 @@ pub mod sorted_map;
 pub mod stable_set;
 #[macro_use]
 pub mod stable_hasher;
+mod atomic_ref;
+pub mod fingerprint;
+pub mod profiling;
 pub mod sharded;
 pub mod stack;
 pub mod sync;
 pub mod thin_vec;
 pub mod tiny_list;
 pub mod transitive_relation;
-pub use ena::undo_log;
-pub use ena::unify;
-mod atomic_ref;
-pub mod fingerprint;
-pub mod profiling;
 pub mod vec_linked_list;
 pub mod work_queue;
 pub use atomic_ref::AtomicRef;
 pub mod frozen;
+pub mod sso;
 pub mod tagged_ptr;
 pub mod temp_dir;
 pub mod unhash;
 
+pub use ena::undo_log;
+pub use ena::unify;
+
 pub struct OnDrop<F: Fn()>(pub F);
 
 impl<F: Fn()> OnDrop<F> {
diff --git a/compiler/rustc_data_structures/src/macros.rs b/compiler/rustc_data_structures/src/macros.rs
index 67fbe3058cd..b918ed9458c 100644
--- a/compiler/rustc_data_structures/src/macros.rs
+++ b/compiler/rustc_data_structures/src/macros.rs
@@ -1,15 +1,3 @@
-/// A simple static assertion macro.
-#[macro_export]
-#[allow_internal_unstable(type_ascription)]
-macro_rules! static_assert {
-    ($test:expr) => {
-        // Use the bool to access an array such that if the bool is false, the access
-        // is out-of-bounds.
-        #[allow(dead_code)]
-        const _: () = [()][!($test: bool) as usize];
-    };
-}
-
 /// Type size assertion. The first argument is a type and the second argument is its expected size.
 #[macro_export]
 macro_rules! static_assert_size {
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 07d16c6483e..363879cbb1d 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -600,10 +600,7 @@ pub fn print_time_passes_entry(do_it: bool, what: &str, dur: Duration) {
 // Hack up our own formatting for the duration to make it easier for scripts
 // to parse (always use the same number of decimal places and the same unit).
 pub fn duration_to_secs_str(dur: std::time::Duration) -> String {
-    const NANOS_PER_SEC: f64 = 1_000_000_000.0;
-    let secs = dur.as_secs() as f64 + dur.subsec_nanos() as f64 / NANOS_PER_SEC;
-
-    format!("{:.3}", secs)
+    format!("{:.3}", dur.as_secs_f64())
 }
 
 // Memory reporting
diff --git a/compiler/rustc_data_structures/src/sip128.rs b/compiler/rustc_data_structures/src/sip128.rs
index beb28dd0720..2c4eff618c6 100644
--- a/compiler/rustc_data_structures/src/sip128.rs
+++ b/compiler/rustc_data_structures/src/sip128.rs
@@ -125,15 +125,28 @@ impl SipHasher128 {
 
     // A specialized write function for values with size <= 8.
     //
-    // The hashing of multi-byte integers depends on endianness. E.g.:
+    // The input must be zero-extended to 64-bits by the caller. This extension
+    // isn't hashed, but the implementation requires it for correctness.
+    //
+    // This function, given the same integer size and value, has the same effect
+    // on both little- and big-endian hardware. It operates on values without
+    // depending on their sequence in memory, so is independent of endianness.
+    //
+    // However, we want SipHasher128 to be platform-dependent, in order to be
+    // consistent with the platform-dependent SipHasher in libstd. In other
+    // words, we want:
+    //
     // - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])`
     // - big-endian:    `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])`
     //
-    // This function does the right thing for little-endian hardware. On
-    // big-endian hardware `x` must be byte-swapped first to give the right
-    // behaviour. After any byte-swapping, the input must be zero-extended to
-    // 64-bits. The caller is responsible for the byte-swapping and
-    // zero-extension.
+    // Therefore, in order to produce endian-dependent results, SipHasher128's
+    // `write_xxx` Hasher trait methods byte-swap `x` prior to zero-extending.
+    //
+    // If clients of SipHasher128 itself want platform-independent results, they
+    // *also* must byte-swap integer inputs before invoking the `write_xxx`
+    // methods on big-endian hardware (that is, two byte-swaps must occur--one
+    // in the client, and one in SipHasher128). Additionally, they must extend
+    // `usize` and `isize` types to 64 bits on 32-bit systems.
     #[inline]
     fn short_write<T>(&mut self, _x: T, x: u64) {
         let size = mem::size_of::<T>();
diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs
index 80b7fc74756..2e2274a7b77 100644
--- a/compiler/rustc_data_structures/src/sip128/tests.rs
+++ b/compiler/rustc_data_structures/src/sip128/tests.rs
@@ -1,7 +1,6 @@
 use super::*;
 
 use std::hash::{Hash, Hasher};
-use std::{mem, slice};
 
 // Hash just the bytes of the slice, without length prefix
 struct Bytes<'a>(&'a [u8]);
@@ -399,20 +398,55 @@ fn test_hash_no_concat_alias() {
 }
 
 #[test]
-fn test_write_short_works() {
-    let test_usize = 0xd0c0b0a0usize;
+fn test_short_write_works() {
+    let test_u8 = 0xFF_u8;
+    let test_u16 = 0x1122_u16;
+    let test_u32 = 0x22334455_u32;
+    let test_u64 = 0x33445566_778899AA_u64;
+    let test_u128 = 0x11223344_55667788_99AABBCC_DDEEFF77_u128;
+    let test_usize = 0xD0C0B0A0_usize;
+
+    let test_i8 = -1_i8;
+    let test_i16 = -2_i16;
+    let test_i32 = -3_i32;
+    let test_i64 = -4_i64;
+    let test_i128 = -5_i128;
+    let test_isize = -6_isize;
+
     let mut h1 = SipHasher128::new_with_keys(0, 0);
-    h1.write_usize(test_usize);
     h1.write(b"bytes");
     h1.write(b"string");
-    h1.write_u8(0xFFu8);
-    h1.write_u8(0x01u8);
+    h1.write_u8(test_u8);
+    h1.write_u16(test_u16);
+    h1.write_u32(test_u32);
+    h1.write_u64(test_u64);
+    h1.write_u128(test_u128);
+    h1.write_usize(test_usize);
+    h1.write_i8(test_i8);
+    h1.write_i16(test_i16);
+    h1.write_i32(test_i32);
+    h1.write_i64(test_i64);
+    h1.write_i128(test_i128);
+    h1.write_isize(test_isize);
+
     let mut h2 = SipHasher128::new_with_keys(0, 0);
-    h2.write(unsafe {
-        slice::from_raw_parts(&test_usize as *const _ as *const u8, mem::size_of::<usize>())
-    });
     h2.write(b"bytes");
     h2.write(b"string");
-    h2.write(&[0xFFu8, 0x01u8]);
-    assert_eq!(h1.finish128(), h2.finish128());
+    h2.write(&test_u8.to_ne_bytes());
+    h2.write(&test_u16.to_ne_bytes());
+    h2.write(&test_u32.to_ne_bytes());
+    h2.write(&test_u64.to_ne_bytes());
+    h2.write(&test_u128.to_ne_bytes());
+    h2.write(&test_usize.to_ne_bytes());
+    h2.write(&test_i8.to_ne_bytes());
+    h2.write(&test_i16.to_ne_bytes());
+    h2.write(&test_i32.to_ne_bytes());
+    h2.write(&test_i64.to_ne_bytes());
+    h2.write(&test_i128.to_ne_bytes());
+    h2.write(&test_isize.to_ne_bytes());
+
+    let h1_hash = h1.finish128();
+    let h2_hash = h2.finish128();
+
+    assert_eq!(h1_hash, h2_hash);
 }
diff --git a/compiler/rustc_data_structures/src/sorted_map.rs b/compiler/rustc_data_structures/src/sorted_map.rs
index 856eb73e629..4807380595d 100644
--- a/compiler/rustc_data_structures/src/sorted_map.rs
+++ b/compiler/rustc_data_structures/src/sorted_map.rs
@@ -34,7 +34,7 @@ impl<K: Ord, V> SortedMap<K, V> {
     /// and that there are no duplicates.
     #[inline]
     pub fn from_presorted_elements(elements: Vec<(K, V)>) -> SortedMap<K, V> {
-        debug_assert!(elements.windows(2).all(|w| w[0].0 < w[1].0));
+        debug_assert!(elements.array_windows().all(|[fst, snd]| fst.0 < snd.0));
 
         SortedMap { data: elements }
     }
@@ -159,7 +159,7 @@ impl<K: Ord, V> SortedMap<K, V> {
             return;
         }
 
-        debug_assert!(elements.windows(2).all(|w| w[0].0 < w[1].0));
+        debug_assert!(elements.array_windows().all(|[fst, snd]| fst.0 < snd.0));
 
         let start_index = self.lookup_index_for(&elements[0].0);
 
diff --git a/compiler/rustc_data_structures/src/sso/either_iter.rs b/compiler/rustc_data_structures/src/sso/either_iter.rs
new file mode 100644
index 00000000000..af8ffcf4c13
--- /dev/null
+++ b/compiler/rustc_data_structures/src/sso/either_iter.rs
@@ -0,0 +1,75 @@
+use std::fmt;
+use std::iter::ExactSizeIterator;
+use std::iter::FusedIterator;
+use std::iter::Iterator;
+
+/// Iterator which may contain instance of
+/// one of two specific implementations.
+///
+/// Note: For most methods providing custom
+///       implementation may margianlly
+///       improve performance by avoiding
+///       doing Left/Right match on every step
+///       and doing it only once instead.
+#[derive(Clone)]
+pub enum EitherIter<L, R> {
+    Left(L),
+    Right(R),
+}
+
+impl<L, R> Iterator for EitherIter<L, R>
+where
+    L: Iterator,
+    R: Iterator<Item = L::Item>,
+{
+    type Item = L::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            EitherIter::Left(l) => l.next(),
+            EitherIter::Right(r) => r.next(),
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        match self {
+            EitherIter::Left(l) => l.size_hint(),
+            EitherIter::Right(r) => r.size_hint(),
+        }
+    }
+}
+
+impl<L, R> ExactSizeIterator for EitherIter<L, R>
+where
+    L: ExactSizeIterator,
+    R: ExactSizeIterator,
+    EitherIter<L, R>: Iterator,
+{
+    fn len(&self) -> usize {
+        match self {
+            EitherIter::Left(l) => l.len(),
+            EitherIter::Right(r) => r.len(),
+        }
+    }
+}
+
+impl<L, R> FusedIterator for EitherIter<L, R>
+where
+    L: FusedIterator,
+    R: FusedIterator,
+    EitherIter<L, R>: Iterator,
+{
+}
+
+impl<L, R> fmt::Debug for EitherIter<L, R>
+where
+    L: fmt::Debug,
+    R: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            EitherIter::Left(l) => l.fmt(f),
+            EitherIter::Right(r) => r.fmt(f),
+        }
+    }
+}
diff --git a/compiler/rustc_data_structures/src/sso/map.rs b/compiler/rustc_data_structures/src/sso/map.rs
new file mode 100644
index 00000000000..fa510e58314
--- /dev/null
+++ b/compiler/rustc_data_structures/src/sso/map.rs
@@ -0,0 +1,560 @@
+use super::either_iter::EitherIter;
+use crate::fx::FxHashMap;
+use arrayvec::ArrayVec;
+use std::fmt;
+use std::hash::Hash;
+use std::iter::FromIterator;
+use std::ops::Index;
+
+// For pointer-sized arguments arrays
+// are faster than set/map for up to 64
+// arguments.
+//
+// On the other hand such a big array
+// hurts cache performance, makes passing
+// sso structures around very expensive.
+//
+// Biggest performance benefit is gained
+// for reasonably small arrays that stay
+// small in vast majority of cases.
+//
+// '8' is choosen as a sane default, to be
+// reevaluated later.
+//
+// Note: As of now ArrayVec design prevents
+//       us from making it user-customizable.
+const SSO_ARRAY_SIZE: usize = 8;
+
+/// Small-storage-optimized implementation of a map.
+///
+/// Stores elements in a small array up to a certain length
+/// and switches to `HashMap` when that length is exceeded.
+//
+// FIXME: Implements subset of HashMap API.
+//
+// Missing HashMap API:
+//   all hasher-related
+//   try_reserve (unstable)
+//   shrink_to (unstable)
+//   drain_filter (unstable)
+//   into_keys/into_values (unstable)
+//   all raw_entry-related
+//   PartialEq/Eq (requires sorting the array)
+//   Entry::or_insert_with_key (unstable)
+//   Vacant/Occupied entries and related
+//
+// FIXME: In HashMap most methods accepting key reference
+// accept reference to generic `Q` where `K: Borrow<Q>`.
+//
+// However, using this approach in `HashMap::get` apparently
+// breaks inlining and noticeably reduces performance.
+//
+// Performance *should* be the same given that borrow is
+// a NOP in most cases, but in practice that's not the case.
+//
+// Further investigation is required.
+//
+// Affected methods:
+//   SsoHashMap::get
+//   SsoHashMap::get_mut
+//   SsoHashMap::get_entry
+//   SsoHashMap::get_key_value
+//   SsoHashMap::contains_key
+//   SsoHashMap::remove
+//   SsoHashMap::remove_entry
+//   Index::index
+//   SsoHashSet::take
+//   SsoHashSet::get
+//   SsoHashSet::remove
+//   SsoHashSet::contains
+
+#[derive(Clone)]
+pub enum SsoHashMap<K, V> {
+    Array(ArrayVec<[(K, V); SSO_ARRAY_SIZE]>),
+    Map(FxHashMap<K, V>),
+}
+
+impl<K, V> SsoHashMap<K, V> {
+    /// Creates an empty `SsoHashMap`.
+    #[inline]
+    pub fn new() -> Self {
+        SsoHashMap::Array(ArrayVec::new())
+    }
+
+    /// Creates an empty `SsoHashMap` with the specified capacity.
+    pub fn with_capacity(cap: usize) -> Self {
+        if cap <= SSO_ARRAY_SIZE {
+            Self::new()
+        } else {
+            SsoHashMap::Map(FxHashMap::with_capacity_and_hasher(cap, Default::default()))
+        }
+    }
+
+    /// Clears the map, removing all key-value pairs. Keeps the allocated memory
+    /// for reuse.
+    pub fn clear(&mut self) {
+        match self {
+            SsoHashMap::Array(array) => array.clear(),
+            SsoHashMap::Map(map) => map.clear(),
+        }
+    }
+
+    /// Returns the number of elements the map can hold without reallocating.
+    pub fn capacity(&self) -> usize {
+        match self {
+            SsoHashMap::Array(_) => SSO_ARRAY_SIZE,
+            SsoHashMap::Map(map) => map.capacity(),
+        }
+    }
+
+    /// Returns the number of elements in the map.
+    pub fn len(&self) -> usize {
+        match self {
+            SsoHashMap::Array(array) => array.len(),
+            SsoHashMap::Map(map) => map.len(),
+        }
+    }
+
+    /// Returns `true` if the map contains no elements.
+    pub fn is_empty(&self) -> bool {
+        match self {
+            SsoHashMap::Array(array) => array.is_empty(),
+            SsoHashMap::Map(map) => map.is_empty(),
+        }
+    }
+
+    /// An iterator visiting all key-value pairs in arbitrary order.
+    /// The iterator element type is `(&'a K, &'a V)`.
+    #[inline]
+    pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter {
+        self.into_iter()
+    }
+
+    /// An iterator visiting all key-value pairs in arbitrary order,
+    /// with mutable references to the values.
+    /// The iterator element type is `(&'a K, &'a mut V)`.
+    #[inline]
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = (&'_ K, &'_ mut V)> {
+        self.into_iter()
+    }
+
+    /// An iterator visiting all keys in arbitrary order.
+    /// The iterator element type is `&'a K`.
+    pub fn keys(&self) -> impl Iterator<Item = &'_ K> {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(k, _v)| k)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.keys()),
+        }
+    }
+
+    /// An iterator visiting all values in arbitrary order.
+    /// The iterator element type is `&'a V`.
+    pub fn values(&self) -> impl Iterator<Item = &'_ V> {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.iter().map(|(_k, v)| v)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.values()),
+        }
+    }
+
+    /// An iterator visiting all values mutably in arbitrary order.
+    /// The iterator element type is `&'a mut V`.
+    pub fn values_mut(&mut self) -> impl Iterator<Item = &'_ mut V> {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.iter_mut().map(|(_k, v)| v)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.values_mut()),
+        }
+    }
+
+    /// Clears the map, returning all key-value pairs as an iterator. Keeps the
+    /// allocated memory for reuse.
+    pub fn drain(&mut self) -> impl Iterator<Item = (K, V)> + '_ {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.drain(..)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.drain()),
+        }
+    }
+}
+
+impl<K: Eq + Hash, V> SsoHashMap<K, V> {
+    /// Changes underlying storage from array to hashmap
+    /// if array is full.
+    fn migrate_if_full(&mut self) {
+        if let SsoHashMap::Array(array) = self {
+            if array.is_full() {
+                *self = SsoHashMap::Map(array.drain(..).collect());
+            }
+        }
+    }
+
+    /// Reserves capacity for at least `additional` more elements to be inserted
+    /// in the `SsoHashMap`. The collection may reserve more space to avoid
+    /// frequent reallocations.
+    pub fn reserve(&mut self, additional: usize) {
+        match self {
+            SsoHashMap::Array(array) => {
+                if SSO_ARRAY_SIZE < (array.len() + additional) {
+                    let mut map: FxHashMap<K, V> = array.drain(..).collect();
+                    map.reserve(additional);
+                    *self = SsoHashMap::Map(map);
+                }
+            }
+            SsoHashMap::Map(map) => map.reserve(additional),
+        }
+    }
+
+    /// Shrinks the capacity of the map as much as possible. It will drop
+    /// down as much as possible while maintaining the internal rules
+    /// and possibly leaving some space in accordance with the resize policy.
+    pub fn shrink_to_fit(&mut self) {
+        if let SsoHashMap::Map(map) = self {
+            if map.len() <= SSO_ARRAY_SIZE {
+                *self = SsoHashMap::Array(map.drain().collect());
+            } else {
+                map.shrink_to_fit();
+            }
+        }
+    }
+
+    /// Retains only the elements specified by the predicate.
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&K, &mut V) -> bool,
+    {
+        match self {
+            SsoHashMap::Array(array) => array.retain(|(k, v)| f(k, v)),
+            SsoHashMap::Map(map) => map.retain(f),
+        }
+    }
+
+    /// Inserts a key-value pair into the map.
+    ///
+    /// If the map did not have this key present, [`None`] is returned.
+    ///
+    /// If the map did have this key present, the value is updated, and the old
+    /// value is returned. The key is not updated, though; this matters for
+    /// types that can be `==` without being identical. See the [module-level
+    /// documentation] for more.
+    pub fn insert(&mut self, key: K, value: V) -> Option<V> {
+        match self {
+            SsoHashMap::Array(array) => {
+                for (k, v) in array.iter_mut() {
+                    if *k == key {
+                        let old_value = std::mem::replace(v, value);
+                        return Some(old_value);
+                    }
+                }
+                if let Err(error) = array.try_push((key, value)) {
+                    let mut map: FxHashMap<K, V> = array.drain(..).collect();
+                    let (key, value) = error.element();
+                    map.insert(key, value);
+                    *self = SsoHashMap::Map(map);
+                }
+                None
+            }
+            SsoHashMap::Map(map) => map.insert(key, value),
+        }
+    }
+
+    /// Removes a key from the map, returning the value at the key if the key
+    /// was previously in the map.
+    pub fn remove(&mut self, key: &K) -> Option<V> {
+        match self {
+            SsoHashMap::Array(array) => {
+                if let Some(index) = array.iter().position(|(k, _v)| k == key) {
+                    Some(array.swap_remove(index).1)
+                } else {
+                    None
+                }
+            }
+            SsoHashMap::Map(map) => map.remove(key),
+        }
+    }
+
+    /// Removes a key from the map, returning the stored key and value if the
+    /// key was previously in the map.
+    pub fn remove_entry(&mut self, key: &K) -> Option<(K, V)> {
+        match self {
+            SsoHashMap::Array(array) => {
+                if let Some(index) = array.iter().position(|(k, _v)| k == key) {
+                    Some(array.swap_remove(index))
+                } else {
+                    None
+                }
+            }
+            SsoHashMap::Map(map) => map.remove_entry(key),
+        }
+    }
+
+    /// Returns a reference to the value corresponding to the key.
+    pub fn get(&self, key: &K) -> Option<&V> {
+        match self {
+            SsoHashMap::Array(array) => {
+                for (k, v) in array {
+                    if k == key {
+                        return Some(v);
+                    }
+                }
+                None
+            }
+            SsoHashMap::Map(map) => map.get(key),
+        }
+    }
+
+    /// Returns a mutable reference to the value corresponding to the key.
+    pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {
+        match self {
+            SsoHashMap::Array(array) => {
+                for (k, v) in array {
+                    if k == key {
+                        return Some(v);
+                    }
+                }
+                None
+            }
+            SsoHashMap::Map(map) => map.get_mut(key),
+        }
+    }
+
+    /// Returns the key-value pair corresponding to the supplied key.
+    pub fn get_key_value(&self, key: &K) -> Option<(&K, &V)> {
+        match self {
+            SsoHashMap::Array(array) => {
+                for (k, v) in array {
+                    if k == key {
+                        return Some((k, v));
+                    }
+                }
+                None
+            }
+            SsoHashMap::Map(map) => map.get_key_value(key),
+        }
+    }
+
+    /// Returns `true` if the map contains a value for the specified key.
+    pub fn contains_key(&self, key: &K) -> bool {
+        match self {
+            SsoHashMap::Array(array) => array.iter().any(|(k, _v)| k == key),
+            SsoHashMap::Map(map) => map.contains_key(key),
+        }
+    }
+
+    /// Gets the given key's corresponding entry in the map for in-place manipulation.
+    #[inline]
+    pub fn entry(&mut self, key: K) -> Entry<'_, K, V> {
+        Entry { ssomap: self, key }
+    }
+}
+
+impl<K, V> Default for SsoHashMap<K, V> {
+    #[inline]
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<K: Eq + Hash, V> FromIterator<(K, V)> for SsoHashMap<K, V> {
+    fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> SsoHashMap<K, V> {
+        let mut map: SsoHashMap<K, V> = Default::default();
+        map.extend(iter);
+        map
+    }
+}
+
+impl<K: Eq + Hash, V> Extend<(K, V)> for SsoHashMap<K, V> {
+    fn extend<I>(&mut self, iter: I)
+    where
+        I: IntoIterator<Item = (K, V)>,
+    {
+        for (key, value) in iter.into_iter() {
+            self.insert(key, value);
+        }
+    }
+
+    #[inline]
+    fn extend_one(&mut self, (k, v): (K, V)) {
+        self.insert(k, v);
+    }
+
+    fn extend_reserve(&mut self, additional: usize) {
+        match self {
+            SsoHashMap::Array(array) => {
+                if SSO_ARRAY_SIZE < (array.len() + additional) {
+                    let mut map: FxHashMap<K, V> = array.drain(..).collect();
+                    map.extend_reserve(additional);
+                    *self = SsoHashMap::Map(map);
+                }
+            }
+            SsoHashMap::Map(map) => map.extend_reserve(additional),
+        }
+    }
+}
+
+impl<'a, K, V> Extend<(&'a K, &'a V)> for SsoHashMap<K, V>
+where
+    K: Eq + Hash + Copy,
+    V: Copy,
+{
+    fn extend<T: IntoIterator<Item = (&'a K, &'a V)>>(&mut self, iter: T) {
+        self.extend(iter.into_iter().map(|(k, v)| (k.clone(), v.clone())))
+    }
+
+    #[inline]
+    fn extend_one(&mut self, (&k, &v): (&'a K, &'a V)) {
+        self.insert(k, v);
+    }
+
+    #[inline]
+    fn extend_reserve(&mut self, additional: usize) {
+        Extend::<(K, V)>::extend_reserve(self, additional)
+    }
+}
+
+impl<K, V> IntoIterator for SsoHashMap<K, V> {
+    type IntoIter = EitherIter<
+        <ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+        <FxHashMap<K, V> as IntoIterator>::IntoIter,
+    >;
+    type Item = <Self::IntoIter as Iterator>::Item;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter()),
+            SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()),
+        }
+    }
+}
+
+/// adapts Item of array reference iterator to Item of hashmap reference iterator.
+#[inline(always)]
+fn adapt_array_ref_it<K, V>(pair: &'a (K, V)) -> (&'a K, &'a V) {
+    let (a, b) = pair;
+    (a, b)
+}
+
+/// adapts Item of array mut reference iterator to Item of hashmap mut reference iterator.
+#[inline(always)]
+fn adapt_array_mut_it<K, V>(pair: &'a mut (K, V)) -> (&'a K, &'a mut V) {
+    let (a, b) = pair;
+    (a, b)
+}
+
+impl<'a, K, V> IntoIterator for &'a SsoHashMap<K, V> {
+    type IntoIter = EitherIter<
+        std::iter::Map<
+            <&'a ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+            fn(&'a (K, V)) -> (&'a K, &'a V),
+        >,
+        <&'a FxHashMap<K, V> as IntoIterator>::IntoIter,
+    >;
+    type Item = <Self::IntoIter as Iterator>::Item;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_ref_it)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()),
+        }
+    }
+}
+
+impl<'a, K, V> IntoIterator for &'a mut SsoHashMap<K, V> {
+    type IntoIter = EitherIter<
+        std::iter::Map<
+            <&'a mut ArrayVec<[(K, V); 8]> as IntoIterator>::IntoIter,
+            fn(&'a mut (K, V)) -> (&'a K, &'a mut V),
+        >,
+        <&'a mut FxHashMap<K, V> as IntoIterator>::IntoIter,
+    >;
+    type Item = <Self::IntoIter as Iterator>::Item;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            SsoHashMap::Array(array) => EitherIter::Left(array.into_iter().map(adapt_array_mut_it)),
+            SsoHashMap::Map(map) => EitherIter::Right(map.into_iter()),
+        }
+    }
+}
+
+impl<K, V> fmt::Debug for SsoHashMap<K, V>
+where
+    K: fmt::Debug,
+    V: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_map().entries(self.iter()).finish()
+    }
+}
+
+impl<'a, K, V> Index<&'a K> for SsoHashMap<K, V>
+where
+    K: Eq + Hash,
+{
+    type Output = V;
+
+    #[inline]
+    fn index(&self, key: &K) -> &V {
+        self.get(key).expect("no entry found for key")
+    }
+}
+
+/// A view into a single entry in a map.
+pub struct Entry<'a, K, V> {
+    ssomap: &'a mut SsoHashMap<K, V>,
+    key: K,
+}
+
+impl<'a, K: Eq + Hash, V> Entry<'a, K, V> {
+    /// Provides in-place mutable access to an occupied entry before any
+    /// potential inserts into the map.
+    pub fn and_modify<F>(self, f: F) -> Self
+    where
+        F: FnOnce(&mut V),
+    {
+        if let Some(value) = self.ssomap.get_mut(&self.key) {
+            f(value);
+        }
+        self
+    }
+
+    /// Ensures a value is in the entry by inserting the default if empty, and returns
+    /// a mutable reference to the value in the entry.
+    #[inline]
+    pub fn or_insert(self, value: V) -> &'a mut V {
+        self.or_insert_with(|| value)
+    }
+
+    /// Ensures a value is in the entry by inserting the result of the default function if empty,
+    /// and returns a mutable reference to the value in the entry.
+    pub fn or_insert_with<F: FnOnce() -> V>(self, default: F) -> &'a mut V {
+        self.ssomap.migrate_if_full();
+        match self.ssomap {
+            SsoHashMap::Array(array) => {
+                let key_ref = &self.key;
+                let found_index = array.iter().position(|(k, _v)| k == key_ref);
+                let index = if let Some(index) = found_index {
+                    index
+                } else {
+                    let index = array.len();
+                    array.try_push((self.key, default())).unwrap();
+                    index
+                };
+                &mut array[index].1
+            }
+            SsoHashMap::Map(map) => map.entry(self.key).or_insert_with(default),
+        }
+    }
+
+    /// Returns a reference to this entry's key.
+    #[inline]
+    pub fn key(&self) -> &K {
+        &self.key
+    }
+}
+
+impl<'a, K: Eq + Hash, V: Default> Entry<'a, K, V> {
+    /// Ensures a value is in the entry by inserting the default value if empty,
+    /// and returns a mutable reference to the value in the entry.
+    #[inline]
+    pub fn or_default(self) -> &'a mut V {
+        self.or_insert_with(Default::default)
+    }
+}
diff --git a/compiler/rustc_data_structures/src/sso/mod.rs b/compiler/rustc_data_structures/src/sso/mod.rs
new file mode 100644
index 00000000000..dd21bc8e696
--- /dev/null
+++ b/compiler/rustc_data_structures/src/sso/mod.rs
@@ -0,0 +1,6 @@
+mod either_iter;
+mod map;
+mod set;
+
+pub use map::SsoHashMap;
+pub use set::SsoHashSet;
diff --git a/compiler/rustc_data_structures/src/sso/set.rs b/compiler/rustc_data_structures/src/sso/set.rs
new file mode 100644
index 00000000000..23cff0206c5
--- /dev/null
+++ b/compiler/rustc_data_structures/src/sso/set.rs
@@ -0,0 +1,237 @@
+use std::fmt;
+use std::hash::Hash;
+use std::iter::FromIterator;
+
+use super::map::SsoHashMap;
+
+/// Small-storage-optimized implementation of a set.
+///
+/// Stores elements in a small array up to a certain length
+/// and switches to `HashSet` when that length is exceeded.
+//
+// FIXME: Implements subset of HashSet API.
+//
+// Missing HashSet API:
+//   all hasher-related
+//   try_reserve (unstable)
+//   shrink_to (unstable)
+//   drain_filter (unstable)
+//   replace
+//   get_or_insert/get_or_insert_owned/get_or_insert_with (unstable)
+//   difference/symmetric_difference/intersection/union
+//   is_disjoint/is_subset/is_superset
+//   PartialEq/Eq (requires SsoHashMap implementation)
+//   BitOr/BitAnd/BitXor/Sub
+#[derive(Clone)]
+pub struct SsoHashSet<T> {
+    map: SsoHashMap<T, ()>,
+}
+
+/// Adapter function used ot return
+/// result if SsoHashMap functions into
+/// result SsoHashSet should return.
+#[inline(always)]
+fn entry_to_key<K, V>((k, _v): (K, V)) -> K {
+    k
+}
+
+impl<T> SsoHashSet<T> {
+    /// Creates an empty `SsoHashSet`.
+    #[inline]
+    pub fn new() -> Self {
+        Self { map: SsoHashMap::new() }
+    }
+
+    /// Creates an empty `SsoHashSet` with the specified capacity.
+    #[inline]
+    pub fn with_capacity(cap: usize) -> Self {
+        Self { map: SsoHashMap::with_capacity(cap) }
+    }
+
+    /// Clears the set, removing all values.
+    #[inline]
+    pub fn clear(&mut self) {
+        self.map.clear()
+    }
+
+    /// Returns the number of elements the set can hold without reallocating.
+    #[inline]
+    pub fn capacity(&self) -> usize {
+        self.map.capacity()
+    }
+
+    /// Returns the number of elements in the set.
+    #[inline]
+    pub fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    /// Returns `true` if the set contains no elements.
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.map.is_empty()
+    }
+
+    /// An iterator visiting all elements in arbitrary order.
+    /// The iterator element type is `&'a T`.
+    #[inline]
+    pub fn iter(&'a self) -> impl Iterator<Item = &'a T> {
+        self.into_iter()
+    }
+
+    /// Clears the set, returning all elements in an iterator.
+    #[inline]
+    pub fn drain(&mut self) -> impl Iterator<Item = T> + '_ {
+        self.map.drain().map(entry_to_key)
+    }
+}
+
+impl<T: Eq + Hash> SsoHashSet<T> {
+    /// Reserves capacity for at least `additional` more elements to be inserted
+    /// in the `SsoHashSet`. The collection may reserve more space to avoid
+    /// frequent reallocations.
+    #[inline]
+    pub fn reserve(&mut self, additional: usize) {
+        self.map.reserve(additional)
+    }
+
+    /// Shrinks the capacity of the set as much as possible. It will drop
+    /// down as much as possible while maintaining the internal rules
+    /// and possibly leaving some space in accordance with the resize policy.
+    #[inline]
+    pub fn shrink_to_fit(&mut self) {
+        self.map.shrink_to_fit()
+    }
+
+    /// Retains only the elements specified by the predicate.
+    #[inline]
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&T) -> bool,
+    {
+        self.map.retain(|k, _v| f(k))
+    }
+
+    /// Removes and returns the value in the set, if any, that is equal to the given one.
+    #[inline]
+    pub fn take(&mut self, value: &T) -> Option<T> {
+        self.map.remove_entry(value).map(entry_to_key)
+    }
+
+    /// Returns a reference to the value in the set, if any, that is equal to the given value.
+    #[inline]
+    pub fn get(&self, value: &T) -> Option<&T> {
+        self.map.get_key_value(value).map(entry_to_key)
+    }
+
+    /// Adds a value to the set.
+    ///
+    /// If the set did not have this value present, `true` is returned.
+    ///
+    /// If the set did have this value present, `false` is returned.
+    #[inline]
+    pub fn insert(&mut self, elem: T) -> bool {
+        self.map.insert(elem, ()).is_none()
+    }
+
+    /// Removes a value from the set. Returns whether the value was
+    /// present in the set.
+    #[inline]
+    pub fn remove(&mut self, value: &T) -> bool {
+        self.map.remove(value).is_some()
+    }
+
+    /// Returns `true` if the set contains a value.
+    #[inline]
+    pub fn contains(&self, value: &T) -> bool {
+        self.map.contains_key(value)
+    }
+}
+
+impl<T: Eq + Hash> FromIterator<T> for SsoHashSet<T> {
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> SsoHashSet<T> {
+        let mut set: SsoHashSet<T> = Default::default();
+        set.extend(iter);
+        set
+    }
+}
+
+impl<T> Default for SsoHashSet<T> {
+    #[inline]
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<T: Eq + Hash> Extend<T> for SsoHashSet<T> {
+    fn extend<I>(&mut self, iter: I)
+    where
+        I: IntoIterator<Item = T>,
+    {
+        for val in iter.into_iter() {
+            self.insert(val);
+        }
+    }
+
+    #[inline]
+    fn extend_one(&mut self, item: T) {
+        self.insert(item);
+    }
+
+    #[inline]
+    fn extend_reserve(&mut self, additional: usize) {
+        self.map.extend_reserve(additional)
+    }
+}
+
+impl<'a, T> Extend<&'a T> for SsoHashSet<T>
+where
+    T: 'a + Eq + Hash + Copy,
+{
+    #[inline]
+    fn extend<I: IntoIterator<Item = &'a T>>(&mut self, iter: I) {
+        self.extend(iter.into_iter().cloned());
+    }
+
+    #[inline]
+    fn extend_one(&mut self, &item: &'a T) {
+        self.insert(item);
+    }
+
+    #[inline]
+    fn extend_reserve(&mut self, additional: usize) {
+        Extend::<T>::extend_reserve(self, additional)
+    }
+}
+
+impl<T> IntoIterator for SsoHashSet<T> {
+    type IntoIter = std::iter::Map<<SsoHashMap<T, ()> as IntoIterator>::IntoIter, fn((T, ())) -> T>;
+    type Item = <Self::IntoIter as Iterator>::Item;
+
+    #[inline]
+    fn into_iter(self) -> Self::IntoIter {
+        self.map.into_iter().map(entry_to_key)
+    }
+}
+
+impl<'a, T> IntoIterator for &'a SsoHashSet<T> {
+    type IntoIter = std::iter::Map<
+        <&'a SsoHashMap<T, ()> as IntoIterator>::IntoIter,
+        fn((&'a T, &'a ())) -> &'a T,
+    >;
+    type Item = <Self::IntoIter as Iterator>::Item;
+
+    #[inline]
+    fn into_iter(self) -> Self::IntoIter {
+        self.map.iter().map(entry_to_key)
+    }
+}
+
+impl<T> fmt::Debug for SsoHashSet<T>
+where
+    T: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_set().entries(self.iter()).finish()
+    }
+}
diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs
index c1c79b174f4..68875b3fbde 100644
--- a/compiler/rustc_data_structures/src/stable_hasher.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher.rs
@@ -5,6 +5,9 @@ use smallvec::SmallVec;
 use std::hash::{BuildHasher, Hash, Hasher};
 use std::mem;
 
+#[cfg(test)]
+mod tests;
+
 /// When hashing something that ends up affecting properties like symbol names,
 /// we want these symbol names to be calculated independently of other factors
 /// like what architecture you're compiling *from*.
@@ -129,7 +132,8 @@ impl Hasher for StableHasher {
     fn write_isize(&mut self, i: isize) {
         // Always treat isize as i64 so we get the same results on 32 and 64 bit
         // platforms. This is important for symbol hashes when cross compiling,
-        // for example.
+        // for example. Sign extending here is preferable as it means that the
+        // same negative number hashes the same on both 32 and 64 bit platforms.
         self.state.write_i64((i as i64).to_le());
     }
 }
diff --git a/compiler/rustc_data_structures/src/stable_hasher/tests.rs b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
new file mode 100644
index 00000000000..cd6ff96a555
--- /dev/null
+++ b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
@@ -0,0 +1,73 @@
+use super::*;
+
+// The tests below compare the computed hashes to particular expected values
+// in order to test that we produce the same results on different platforms,
+// regardless of endianness and `usize` and `isize` size differences (this
+// of course assumes we run these tests on platforms that differ in those
+// ways). The expected values depend on the hashing algorithm used, so they
+// need to be updated whenever StableHasher changes its hashing algorithm.
+
+#[test]
+fn test_hash_integers() {
+    // Test that integers are handled consistently across platforms.
+    let test_u8 = 0xAB_u8;
+    let test_u16 = 0xFFEE_u16;
+    let test_u32 = 0x445577AA_u32;
+    let test_u64 = 0x01234567_13243546_u64;
+    let test_u128 = 0x22114433_66557788_99AACCBB_EEDDFF77_u128;
+    let test_usize = 0xD0C0B0A0_usize;
+
+    let test_i8 = -100_i8;
+    let test_i16 = -200_i16;
+    let test_i32 = -300_i32;
+    let test_i64 = -400_i64;
+    let test_i128 = -500_i128;
+    let test_isize = -600_isize;
+
+    let mut h = StableHasher::new();
+    test_u8.hash(&mut h);
+    test_u16.hash(&mut h);
+    test_u32.hash(&mut h);
+    test_u64.hash(&mut h);
+    test_u128.hash(&mut h);
+    test_usize.hash(&mut h);
+    test_i8.hash(&mut h);
+    test_i16.hash(&mut h);
+    test_i32.hash(&mut h);
+    test_i64.hash(&mut h);
+    test_i128.hash(&mut h);
+    test_isize.hash(&mut h);
+
+    // This depends on the hashing algorithm. See note at top of file.
+    let expected = (2736651863462566372, 8121090595289675650);
+
+    assert_eq!(h.finalize(), expected);
+}
+
+#[test]
+fn test_hash_usize() {
+    // Test that usize specifically is handled consistently across platforms.
+    let test_usize = 0xABCDEF01_usize;
+
+    let mut h = StableHasher::new();
+    test_usize.hash(&mut h);
+
+    // This depends on the hashing algorithm. See note at top of file.
+    let expected = (5798740672699530587, 11186240177685111648);
+
+    assert_eq!(h.finalize(), expected);
+}
+
+#[test]
+fn test_hash_isize() {
+    // Test that isize specifically is handled consistently across platforms.
+    let test_isize = -7_isize;
+
+    let mut h = StableHasher::new();
+    test_isize.hash(&mut h);
+
+    // This depends on the hashing algorithm. See note at top of file.
+    let expected = (14721296605626097289, 11385941877786388409);
+
+    assert_eq!(h.finalize(), expected);
+}
diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
index d39d146db31..d63bcdb3c2b 100644
--- a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
+++ b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs
@@ -48,7 +48,7 @@ where
     P: Pointer,
     T: Tag,
 {
-    const TAG_BIT_SHIFT: usize = (8 * std::mem::size_of::<usize>()) - T::BITS;
+    const TAG_BIT_SHIFT: usize = usize::BITS as usize - T::BITS;
     const ASSERTION: () = {
         assert!(T::BITS <= P::BITS);
         // Used for the transmute_copy's below
diff --git a/compiler/rustc_data_structures/src/temp_dir.rs b/compiler/rustc_data_structures/src/temp_dir.rs
index 0d9b3e3ca25..a780d2386a6 100644
--- a/compiler/rustc_data_structures/src/temp_dir.rs
+++ b/compiler/rustc_data_structures/src/temp_dir.rs
@@ -12,7 +12,7 @@ pub struct MaybeTempDir {
 
 impl Drop for MaybeTempDir {
     fn drop(&mut self) {
-        // Safety: We are in the destructor, and no further access will
+        // SAFETY: We are in the destructor, and no further access will
         // occur.
         let dir = unsafe { ManuallyDrop::take(&mut self.dir) };
         if self.keep {