Merge remote-tracking branch 'upstream/master' into box-alloc

7 files changed, 719 insertions, 579 deletions

diff --git a/compiler/rustc_data_structures/src/graph/iterate/mod.rs b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
index bc3d1ce53ba..5f42d46e285 100644
--- a/compiler/rustc_data_structures/src/graph/iterate/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/iterate/mod.rs
@@ -1,6 +1,7 @@
 use super::{DirectedGraph, WithNumNodes, WithStartNode, WithSuccessors};
 use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
+use std::ops::ControlFlow;
 
 #[cfg(test)]
 mod tests;
@@ -86,10 +87,6 @@ where
     }
 }
 
-/// Allows searches to terminate early with a value.
-// FIXME (#75744): remove the alias once the generics are in a better order and `C=()`.
-pub type ControlFlow<T> = std::ops::ControlFlow<(), T>;
-
 /// The status of a node in the depth-first search.
 ///
 /// See the documentation of `TriColorDepthFirstSearch` to see how a node's status is updated
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 9958e5dd5e0..7669b78834c 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -28,6 +28,7 @@
 #![feature(const_panic)]
 #![feature(min_const_generics)]
 #![feature(once_cell)]
+#![feature(maybe_uninit_uninit_array)]
 #![allow(rustc::default_hash_types)]
 
 #[macro_use]
diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
index c0193e9fa0c..a5b2df1da5d 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
@@ -149,8 +149,8 @@ pub struct ObligationForest<O: ForestObligation> {
     /// comments in `process_obligation` for details.
     active_cache: FxHashMap<O::CacheKey, usize>,
 
-    /// A vector reused in compress(), to avoid allocating new vectors.
-    node_rewrites: Vec<usize>,
+    /// A vector reused in compress() and find_cycles_from_node(), to avoid allocating new vectors.
+    reused_node_vec: Vec<usize>,
 
     obligation_tree_id_generator: ObligationTreeIdGenerator,
 
@@ -251,12 +251,22 @@ enum NodeState {
     Error,
 }
 
+/// This trait allows us to have two different Outcome types:
+///  - the normal one that does as little as possible
+///  - one for tests that does some additional work and checking
+pub trait OutcomeTrait {
+    type Error;
+    type Obligation;
+
+    fn new() -> Self;
+    fn mark_not_stalled(&mut self);
+    fn is_stalled(&self) -> bool;
+    fn record_completed(&mut self, outcome: &Self::Obligation);
+    fn record_error(&mut self, error: Self::Error);
+}
+
 #[derive(Debug)]
 pub struct Outcome<O, E> {
-    /// Obligations that were completely evaluated, including all
-    /// (transitive) subobligations. Only computed if requested.
-    pub completed: Option<Vec<O>>,
-
     /// Backtrace of obligations that were found to be in error.
     pub errors: Vec<Error<O, E>>,
 
@@ -269,12 +279,29 @@ pub struct Outcome<O, E> {
     pub stalled: bool,
 }
 
-/// Should `process_obligations` compute the `Outcome::completed` field of its
-/// result?
-#[derive(PartialEq)]
-pub enum DoCompleted {
-    No,
-    Yes,
+impl<O, E> OutcomeTrait for Outcome<O, E> {
+    type Error = Error<O, E>;
+    type Obligation = O;
+
+    fn new() -> Self {
+        Self { stalled: true, errors: vec![] }
+    }
+
+    fn mark_not_stalled(&mut self) {
+        self.stalled = false;
+    }
+
+    fn is_stalled(&self) -> bool {
+        self.stalled
+    }
+
+    fn record_completed(&mut self, _outcome: &Self::Obligation) {
+        // do nothing
+    }
+
+    fn record_error(&mut self, error: Self::Error) {
+        self.errors.push(error)
+    }
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -289,7 +316,7 @@ impl<O: ForestObligation> ObligationForest<O> {
             nodes: vec![],
             done_cache: Default::default(),
             active_cache: Default::default(),
-            node_rewrites: vec![],
+            reused_node_vec: vec![],
             obligation_tree_id_generator: (0..).map(ObligationTreeId),
             error_cache: Default::default(),
         }
@@ -363,8 +390,7 @@ impl<O: ForestObligation> ObligationForest<O> {
             .map(|(index, _node)| Error { error: error.clone(), backtrace: self.error_at(index) })
             .collect();
 
-        let successful_obligations = self.compress(DoCompleted::Yes);
-        assert!(successful_obligations.unwrap().is_empty());
+        self.compress(|_| assert!(false));
         errors
     }
 
@@ -392,16 +418,12 @@ impl<O: ForestObligation> ObligationForest<O> {
     /// be called in a loop until `outcome.stalled` is false.
     ///
     /// This _cannot_ be unrolled (presently, at least).
-    pub fn process_obligations<P>(
-        &mut self,
-        processor: &mut P,
-        do_completed: DoCompleted,
-    ) -> Outcome<O, P::Error>
+    pub fn process_obligations<P, OUT>(&mut self, processor: &mut P) -> OUT
     where
         P: ObligationProcessor<Obligation = O>,
+        OUT: OutcomeTrait<Obligation = O, Error = Error<O, P::Error>>,
     {
-        let mut errors = vec![];
-        let mut stalled = true;
+        let mut outcome = OUT::new();
 
         // Note that the loop body can append new nodes, and those new nodes
         // will then be processed by subsequent iterations of the loop.
@@ -429,7 +451,7 @@ impl<O: ForestObligation> ObligationForest<O> {
                 }
                 ProcessResult::Changed(children) => {
                     // We are not (yet) stalled.
-                    stalled = false;
+                    outcome.mark_not_stalled();
                     node.state.set(NodeState::Success);
 
                     for child in children {
@@ -442,28 +464,22 @@ impl<O: ForestObligation> ObligationForest<O> {
                     }
                 }
                 ProcessResult::Error(err) => {
-                    stalled = false;
-                    errors.push(Error { error: err, backtrace: self.error_at(index) });
+                    outcome.mark_not_stalled();
+                    outcome.record_error(Error { error: err, backtrace: self.error_at(index) });
                 }
             }
             index += 1;
         }
 
-        if stalled {
-            // There's no need to perform marking, cycle processing and compression when nothing
-            // changed.
-            return Outcome {
-                completed: if do_completed == DoCompleted::Yes { Some(vec![]) } else { None },
-                errors,
-                stalled,
-            };
+        // There's no need to perform marking, cycle processing and compression when nothing
+        // changed.
+        if !outcome.is_stalled() {
+            self.mark_successes();
+            self.process_cycles(processor);
+            self.compress(|obl| outcome.record_completed(obl));
         }
 
-        self.mark_successes();
-        self.process_cycles(processor);
-        let completed = self.compress(do_completed);
-
-        Outcome { completed, errors, stalled }
+        outcome
     }
 
     /// Returns a vector of obligations for `p` and all of its
@@ -526,7 +542,6 @@ impl<O: ForestObligation> ObligationForest<O> {
             let node = &self.nodes[index];
             let state = node.state.get();
             if state == NodeState::Success {
-                node.state.set(NodeState::Waiting);
                 // This call site is cold.
                 self.uninlined_mark_dependents_as_waiting(node);
             } else {
@@ -538,17 +553,18 @@ impl<O: ForestObligation> ObligationForest<O> {
     // This never-inlined function is for the cold call site.
     #[inline(never)]
     fn uninlined_mark_dependents_as_waiting(&self, node: &Node<O>) {
+        // Mark node Waiting in the cold uninlined code instead of the hot inlined
+        node.state.set(NodeState::Waiting);
         self.inlined_mark_dependents_as_waiting(node)
     }
 
     /// Report cycles between all `Success` nodes, and convert all `Success`
     /// nodes to `Done`. This must be called after `mark_successes`.
-    fn process_cycles<P>(&self, processor: &mut P)
+    fn process_cycles<P>(&mut self, processor: &mut P)
     where
         P: ObligationProcessor<Obligation = O>,
     {
-        let mut stack = vec![];
-
+        let mut stack = std::mem::take(&mut self.reused_node_vec);
         for (index, node) in self.nodes.iter().enumerate() {
             // For some benchmarks this state test is extremely hot. It's a win
             // to handle the no-op cases immediately to avoid the cost of the
@@ -559,6 +575,7 @@ impl<O: ForestObligation> ObligationForest<O> {
         }
 
         debug_assert!(stack.is_empty());
+        self.reused_node_vec = stack;
     }
 
     fn find_cycles_from_node<P>(&self, stack: &mut Vec<usize>, processor: &mut P, index: usize)
@@ -591,13 +608,12 @@ impl<O: ForestObligation> ObligationForest<O> {
     /// indices and hence invalidates any outstanding indices. `process_cycles`
     /// must be run beforehand to remove any cycles on `Success` nodes.
     #[inline(never)]
-    fn compress(&mut self, do_completed: DoCompleted) -> Option<Vec<O>> {
+    fn compress(&mut self, mut outcome_cb: impl FnMut(&O)) {
         let orig_nodes_len = self.nodes.len();
-        let mut node_rewrites: Vec<_> = std::mem::take(&mut self.node_rewrites);
+        let mut node_rewrites: Vec<_> = std::mem::take(&mut self.reused_node_vec);
         debug_assert!(node_rewrites.is_empty());
         node_rewrites.extend(0..orig_nodes_len);
         let mut dead_nodes = 0;
-        let mut removed_done_obligations: Vec<O> = vec![];
 
         // Move removable nodes to the end, preserving the order of the
         // remaining nodes.
@@ -627,10 +643,8 @@ impl<O: ForestObligation> ObligationForest<O> {
                     } else {
                         self.done_cache.insert(node.obligation.as_cache_key().clone());
                     }
-                    if do_completed == DoCompleted::Yes {
-                        // Extract the success stories.
-                        removed_done_obligations.push(node.obligation.clone());
-                    }
+                    // Extract the success stories.
+                    outcome_cb(&node.obligation);
                     node_rewrites[index] = orig_nodes_len;
                     dead_nodes += 1;
                 }
@@ -654,9 +668,7 @@ impl<O: ForestObligation> ObligationForest<O> {
         }
 
         node_rewrites.truncate(0);
-        self.node_rewrites = node_rewrites;
-
-        if do_completed == DoCompleted::Yes { Some(removed_done_obligations) } else { None }
+        self.reused_node_vec = node_rewrites;
     }
 
     fn apply_rewrites(&mut self, node_rewrites: &[usize]) {
diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs
index 01652465eea..371c62c063f 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs
@@ -17,6 +17,40 @@ struct ClosureObligationProcessor<OF, BF, O, E> {
     marker: PhantomData<(O, E)>,
 }
 
+struct TestOutcome<O, E> {
+    pub completed: Vec<O>,
+    pub errors: Vec<Error<O, E>>,
+    pub stalled: bool,
+}
+
+impl<O, E> OutcomeTrait for TestOutcome<O, E>
+where
+    O: Clone,
+{
+    type Error = Error<O, E>;
+    type Obligation = O;
+
+    fn new() -> Self {
+        Self { errors: vec![], stalled: false, completed: vec![] }
+    }
+
+    fn mark_not_stalled(&mut self) {
+        self.stalled = false;
+    }
+
+    fn is_stalled(&self) -> bool {
+        self.stalled
+    }
+
+    fn record_completed(&mut self, outcome: &Self::Obligation) {
+        self.completed.push(outcome.clone())
+    }
+
+    fn record_error(&mut self, error: Self::Error) {
+        self.errors.push(error)
+    }
+}
+
 #[allow(non_snake_case)]
 fn C<OF, BF, O>(of: OF, bf: BF) -> ClosureObligationProcessor<OF, BF, O, &'static str>
 where
@@ -65,20 +99,17 @@ fn push_pop() {
     //      A |-> A.1
     //        |-> A.2
     //        |-> A.3
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
-                "B" => ProcessResult::Error("B is for broken"),
-                "C" => ProcessResult::Changed(vec![]),
-                "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap(), vec!["C"]);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
+            "B" => ProcessResult::Error("B is for broken"),
+            "C" => ProcessResult::Changed(vec![]),
+            "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok, vec!["C"]);
     assert_eq!(err, vec![Error { error: "B is for broken", backtrace: vec!["B"] }]);
 
     // second round: two delays, one success, creating an uneven set of subtasks:
@@ -88,60 +119,51 @@ fn push_pop() {
     //      D |-> D.1
     //        |-> D.2
     forest.register_obligation("D");
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.1" => ProcessResult::Unchanged,
-                "A.2" => ProcessResult::Unchanged,
-                "A.3" => ProcessResult::Changed(vec!["A.3.i"]),
-                "D" => ProcessResult::Changed(vec!["D.1", "D.2"]),
-                "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap(), Vec::<&'static str>::new());
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.1" => ProcessResult::Unchanged,
+            "A.2" => ProcessResult::Unchanged,
+            "A.3" => ProcessResult::Changed(vec!["A.3.i"]),
+            "D" => ProcessResult::Changed(vec!["D.1", "D.2"]),
+            "A.3.i" | "D.1" | "D.2" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok, Vec::<&'static str>::new());
     assert_eq!(err, Vec::new());
 
     // third round: ok in A.1 but trigger an error in A.2. Check that it
     // propagates to A, but not D.1 or D.2.
     //      D |-> D.1 |-> D.1.i
     //        |-> D.2 |-> D.2.i
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.1" => ProcessResult::Changed(vec![]),
-                "A.2" => ProcessResult::Error("A is for apple"),
-                "A.3.i" => ProcessResult::Changed(vec![]),
-                "D.1" => ProcessResult::Changed(vec!["D.1.i"]),
-                "D.2" => ProcessResult::Changed(vec!["D.2.i"]),
-                "D.1.i" | "D.2.i" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.1" => ProcessResult::Changed(vec![]),
+            "A.2" => ProcessResult::Error("A is for apple"),
+            "A.3.i" => ProcessResult::Changed(vec![]),
+            "D.1" => ProcessResult::Changed(vec!["D.1.i"]),
+            "D.2" => ProcessResult::Changed(vec!["D.2.i"]),
+            "D.1.i" | "D.2.i" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["A.1", "A.3", "A.3.i"]);
     assert_eq!(err, vec![Error { error: "A is for apple", backtrace: vec!["A.2", "A"] }]);
 
     // fourth round: error in D.1.i
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D.1.i" => ProcessResult::Error("D is for dumb"),
-                "D.2.i" => ProcessResult::Changed(vec![]),
-                _ => panic!("unexpected obligation {:?}", obligation),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D.1.i" => ProcessResult::Error("D is for dumb"),
+            "D.2.i" => ProcessResult::Changed(vec![]),
+            _ => panic!("unexpected obligation {:?}", obligation),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["D.2", "D.2.i"]);
     assert_eq!(err, vec![Error { error: "D is for dumb", backtrace: vec!["D.1.i", "D.1", "D"] }]);
@@ -160,72 +182,60 @@ fn success_in_grandchildren() {
     let mut forest = ObligationForest::new();
     forest.register_obligation("A");
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
-                "A.1" => ProcessResult::Changed(vec![]),
-                "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]),
-                "A.3" => ProcessResult::Changed(vec![]),
-                "A.2.i" | "A.2.ii" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
+            "A.1" => ProcessResult::Changed(vec![]),
+            "A.2" => ProcessResult::Changed(vec!["A.2.i", "A.2.ii"]),
+            "A.3" => ProcessResult::Changed(vec![]),
+            "A.2.i" | "A.2.ii" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["A.1", "A.3"]);
     assert!(err.is_empty());
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.2.i" => ProcessResult::Unchanged,
-                "A.2.ii" => ProcessResult::Changed(vec![]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap(), vec!["A.2.ii"]);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.2.i" => ProcessResult::Unchanged,
+            "A.2.ii" => ProcessResult::Changed(vec![]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok, vec!["A.2.ii"]);
     assert!(err.is_empty());
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]),
-                "A.2.i.a" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert!(ok.unwrap().is_empty());
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.2.i" => ProcessResult::Changed(vec!["A.2.i.a"]),
+            "A.2.i.a" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert!(ok.is_empty());
     assert!(err.is_empty());
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.2.i.a" => ProcessResult::Changed(vec![]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.2.i.a" => ProcessResult::Changed(vec![]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["A", "A.2", "A.2.i", "A.2.i.a"]);
     assert!(err.is_empty());
 
-    let Outcome { completed: ok, errors: err, .. } =
-        forest.process_obligations(&mut C(|_| unreachable!(), |_| {}), DoCompleted::Yes);
+    let TestOutcome { completed: ok, errors: err, .. } =
+        forest.process_obligations(&mut C(|_| unreachable!(), |_| {}));
 
-    assert!(ok.unwrap().is_empty());
+    assert!(ok.is_empty());
     assert!(err.is_empty());
 }
 
@@ -235,18 +245,15 @@ fn to_errors_no_throw() {
     // yields to correct errors (and does not panic, in particular).
     let mut forest = ObligationForest::new();
     forest.register_obligation("A");
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
-                "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Changed(vec!["A.1", "A.2", "A.3"]),
+            "A.1" | "A.2" | "A.3" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
     let errors = forest.to_errors(());
     assert_eq!(errors[0].backtrace, vec!["A.1", "A"]);
@@ -260,51 +267,42 @@ fn diamond() {
     // check that diamond dependencies are handled correctly
     let mut forest = ObligationForest::new();
     forest.register_obligation("A");
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Changed(vec!["A.1", "A.2"]),
-                "A.1" | "A.2" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Changed(vec!["A.1", "A.2"]),
+            "A.1" | "A.2" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A.1" => ProcessResult::Changed(vec!["D"]),
-                "A.2" => ProcessResult::Changed(vec!["D"]),
-                "D" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A.1" => ProcessResult::Changed(vec!["D"]),
+            "A.2" => ProcessResult::Changed(vec!["D"]),
+            "D" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
 
     let mut d_count = 0;
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D" => {
-                    d_count += 1;
-                    ProcessResult::Changed(vec![])
-                }
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D" => {
+                d_count += 1;
+                ProcessResult::Changed(vec![])
+            }
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
     assert_eq!(d_count, 1);
-    let mut ok = ok.unwrap();
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["A", "A.1", "A.2", "D"]);
     assert_eq!(err.len(), 0);
@@ -313,51 +311,42 @@ fn diamond() {
     assert_eq!(errors.len(), 0);
 
     forest.register_obligation("A'");
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]),
-                "A'.1" | "A'.2" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A'" => ProcessResult::Changed(vec!["A'.1", "A'.2"]),
+            "A'.1" | "A'.2" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]),
-                "A'.2" => ProcessResult::Changed(vec!["D'"]),
-                "D'" | "A'" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A'.1" => ProcessResult::Changed(vec!["D'", "A'"]),
+            "A'.2" => ProcessResult::Changed(vec!["D'"]),
+            "D'" | "A'" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
 
     let mut d_count = 0;
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D'" => {
-                    d_count += 1;
-                    ProcessResult::Error("operation failed")
-                }
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D'" => {
+                d_count += 1;
+                ProcessResult::Error("operation failed")
+            }
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
     assert_eq!(d_count, 1);
-    assert_eq!(ok.unwrap().len(), 0);
+    assert_eq!(ok.len(), 0);
     assert_eq!(
         err,
         vec![super::Error { error: "operation failed", backtrace: vec!["D'", "A'.1", "A'"] }]
@@ -375,35 +364,27 @@ fn done_dependency() {
     forest.register_obligation("B: Sized");
     forest.register_obligation("C: Sized");
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A: Sized" | "B: Sized" | "C: Sized" => ProcessResult::Changed(vec![]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A: Sized" | "B: Sized" | "C: Sized" => ProcessResult::Changed(vec![]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["A: Sized", "B: Sized", "C: Sized"]);
     assert_eq!(err.len(), 0);
 
     forest.register_obligation("(A,B,C): Sized");
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "(A,B,C): Sized" => {
-                    ProcessResult::Changed(vec!["A: Sized", "B: Sized", "C: Sized"])
-                }
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap(), vec!["(A,B,C): Sized"]);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "(A,B,C): Sized" => ProcessResult::Changed(vec!["A: Sized", "B: Sized", "C: Sized"]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok, vec!["(A,B,C): Sized"]);
     assert_eq!(err.len(), 0);
 }
 
@@ -416,64 +397,52 @@ fn orphan() {
     forest.register_obligation("C1");
     forest.register_obligation("C2");
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Changed(vec!["D", "E"]),
-                "B" => ProcessResult::Unchanged,
-                "C1" => ProcessResult::Changed(vec![]),
-                "C2" => ProcessResult::Changed(vec![]),
-                "D" | "E" => ProcessResult::Unchanged,
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    let mut ok = ok.unwrap();
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Changed(vec!["D", "E"]),
+            "B" => ProcessResult::Unchanged,
+            "C1" => ProcessResult::Changed(vec![]),
+            "C2" => ProcessResult::Changed(vec![]),
+            "D" | "E" => ProcessResult::Unchanged,
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    let mut ok = ok;
     ok.sort();
     assert_eq!(ok, vec!["C1", "C2"]);
     assert_eq!(err.len(), 0);
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D" | "E" => ProcessResult::Unchanged,
-                "B" => ProcessResult::Changed(vec!["D"]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D" | "E" => ProcessResult::Unchanged,
+            "B" => ProcessResult::Changed(vec!["D"]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err.len(), 0);
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D" => ProcessResult::Unchanged,
-                "E" => ProcessResult::Error("E is for error"),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D" => ProcessResult::Unchanged,
+            "E" => ProcessResult::Error("E is for error"),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err, vec![super::Error { error: "E is for error", backtrace: vec!["E", "A"] }]);
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "D" => ProcessResult::Error("D is dead"),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "D" => ProcessResult::Error("D is dead"),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err, vec![super::Error { error: "D is dead", backtrace: vec!["D"] }]);
 
     let errors = forest.to_errors(());
@@ -487,35 +456,29 @@ fn simultaneous_register_and_error() {
     forest.register_obligation("A");
     forest.register_obligation("B");
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Error("An error"),
-                "B" => ProcessResult::Changed(vec!["A"]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Error("An error"),
+            "B" => ProcessResult::Changed(vec!["A"]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]);
 
     let mut forest = ObligationForest::new();
     forest.register_obligation("B");
     forest.register_obligation("A");
 
-    let Outcome { completed: ok, errors: err, .. } = forest.process_obligations(
-        &mut C(
-            |obligation| match *obligation {
-                "A" => ProcessResult::Error("An error"),
-                "B" => ProcessResult::Changed(vec!["A"]),
-                _ => unreachable!(),
-            },
-            |_| {},
-        ),
-        DoCompleted::Yes,
-    );
-    assert_eq!(ok.unwrap().len(), 0);
+    let TestOutcome { completed: ok, errors: err, .. } = forest.process_obligations(&mut C(
+        |obligation| match *obligation {
+            "A" => ProcessResult::Error("An error"),
+            "B" => ProcessResult::Changed(vec!["A"]),
+            _ => unreachable!(),
+        },
+        |_| {},
+    ));
+    assert_eq!(ok.len(), 0);
     assert_eq!(err, vec![super::Error { error: "An error", backtrace: vec!["A"] }]);
 }
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 363879cbb1d..e598d7a683d 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -94,34 +94,9 @@ use std::process;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use measureme::{EventId, EventIdBuilder, SerializableString, StringId};
+use measureme::{EventId, EventIdBuilder, Profiler, SerializableString, StringId};
 use parking_lot::RwLock;
 
-cfg_if! {
-    if #[cfg(any(windows, target_os = "wasi"))] {
-        /// FileSerializationSink is faster on Windows
-        type SerializationSink = measureme::FileSerializationSink;
-    } else if #[cfg(target_arch = "wasm32")] {
-        type SerializationSink = measureme::ByteVecSink;
-    } else {
-        /// MmapSerializatioSink is faster on macOS and Linux
-        type SerializationSink = measureme::MmapSerializationSink;
-    }
-}
-
-type Profiler = measureme::Profiler<SerializationSink>;
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)]
-pub enum ProfileCategory {
-    Parsing,
-    Expansion,
-    TypeChecking,
-    BorrowChecking,
-    Codegen,
-    Linking,
-    Other,
-}
-
 bitflags::bitflags! {
     struct EventFilter: u32 {
         const GENERIC_ACTIVITIES = 1 << 0;
@@ -400,7 +375,7 @@ impl SelfProfiler {
         output_directory: &Path,
         crate_name: Option<&str>,
         event_filters: &Option<Vec<String>>,
-    ) -> Result<SelfProfiler, Box<dyn Error>> {
+    ) -> Result<SelfProfiler, Box<dyn Error + Send + Sync>> {
         fs::create_dir_all(output_directory)?;
 
         let crate_name = crate_name.unwrap_or("unknown-crate");
@@ -511,13 +486,13 @@ impl SelfProfiler {
         self.event_filter_mask.contains(EventFilter::QUERY_KEYS)
     }
 
-    pub fn event_id_builder(&self) -> EventIdBuilder<'_, SerializationSink> {
+    pub fn event_id_builder(&self) -> EventIdBuilder<'_> {
         EventIdBuilder::new(&self.profiler)
     }
 }
 
 #[must_use]
-pub struct TimingGuard<'a>(Option<measureme::TimingGuard<'a, SerializationSink>>);
+pub struct TimingGuard<'a>(Option<measureme::TimingGuard<'a>>);
 
 impl<'a> TimingGuard<'a> {
     #[inline]
diff --git a/compiler/rustc_data_structures/src/sip128.rs b/compiler/rustc_data_structures/src/sip128.rs
index 2c4eff618c6..53062b9c20d 100644
--- a/compiler/rustc_data_structures/src/sip128.rs
+++ b/compiler/rustc_data_structures/src/sip128.rs
@@ -1,21 +1,53 @@
 //! This is a copy of `core::hash::sip` adapted to providing 128 bit hashes.
 
-use std::cmp;
 use std::hash::Hasher;
-use std::mem;
+use std::mem::{self, MaybeUninit};
 use std::ptr;
 
 #[cfg(test)]
 mod tests;
 
+// The SipHash algorithm operates on 8-byte chunks.
+const ELEM_SIZE: usize = mem::size_of::<u64>();
+
+// Size of the buffer in number of elements, not including the spill.
+//
+// The selection of this size was guided by rustc-perf benchmark comparisons of
+// different buffer sizes. It should be periodically reevaluated as the compiler
+// implementation and input characteristics change.
+//
+// Using the same-sized buffer for everything we hash is a performance versus
+// complexity tradeoff. The ideal buffer size, and whether buffering should even
+// be used, depends on what is being hashed. It may be worth it to size the
+// buffer appropriately (perhaps by making SipHasher128 generic over the buffer
+// size) or disable buffering depending on what is being hashed. But at this
+// time, we use the same buffer size for everything.
+const BUFFER_CAPACITY: usize = 8;
+
+// Size of the buffer in bytes, not including the spill.
+const BUFFER_SIZE: usize = BUFFER_CAPACITY * ELEM_SIZE;
+
+// Size of the buffer in number of elements, including the spill.
+const BUFFER_WITH_SPILL_CAPACITY: usize = BUFFER_CAPACITY + 1;
+
+// Size of the buffer in bytes, including the spill.
+const BUFFER_WITH_SPILL_SIZE: usize = BUFFER_WITH_SPILL_CAPACITY * ELEM_SIZE;
+
+// Index of the spill element in the buffer.
+const BUFFER_SPILL_INDEX: usize = BUFFER_WITH_SPILL_CAPACITY - 1;
+
 #[derive(Debug, Clone)]
+#[repr(C)]
 pub struct SipHasher128 {
-    k0: u64,
-    k1: u64,
-    length: usize, // how many bytes we've processed
-    state: State,  // hash State
-    tail: u64,     // unprocessed bytes le
-    ntail: usize,  // how many bytes in tail are valid
+    // The access pattern during hashing consists of accesses to `nbuf` and
+    // `buf` until the buffer is full, followed by accesses to `state` and
+    // `processed`, and then repetition of that pattern until hashing is done.
+    // This is the basis for the ordering of fields below. However, in practice
+    // the cache miss-rate for data access is extremely low regardless of order.
+    nbuf: usize, // how many bytes in buf are valid
+    buf: [MaybeUninit<u64>; BUFFER_WITH_SPILL_CAPACITY], // unprocessed bytes le
+    state: State, // hash State
+    processed: usize, // how many bytes we've processed
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -51,178 +83,328 @@ macro_rules! compress {
     }};
 }
 
-/// Loads an integer of the desired type from a byte stream, in LE order. Uses
-/// `copy_nonoverlapping` to let the compiler generate the most efficient way
-/// to load it from a possibly unaligned address.
-///
-/// Unsafe because: unchecked indexing at i..i+size_of(int_ty)
-macro_rules! load_int_le {
-    ($buf:expr, $i:expr, $int_ty:ident) => {{
-        debug_assert!($i + mem::size_of::<$int_ty>() <= $buf.len());
-        let mut data = 0 as $int_ty;
-        ptr::copy_nonoverlapping(
-            $buf.get_unchecked($i),
-            &mut data as *mut _ as *mut u8,
-            mem::size_of::<$int_ty>(),
-        );
-        data.to_le()
-    }};
-}
-
-/// Loads a u64 using up to 7 bytes of a byte slice. It looks clumsy but the
-/// `copy_nonoverlapping` calls that occur (via `load_int_le!`) all have fixed
-/// sizes and avoid calling `memcpy`, which is good for speed.
-///
-/// Unsafe because: unchecked indexing at start..start+len
+// Copies up to 8 bytes from source to destination. This performs better than
+// `ptr::copy_nonoverlapping` on microbenchmarks and may perform better on real
+// workloads since all of the copies have fixed sizes and avoid calling memcpy.
+//
+// This is specifically designed for copies of up to 8 bytes, because that's the
+// maximum of number bytes needed to fill an 8-byte-sized element on which
+// SipHash operates. Note that for variable-sized copies which are known to be
+// less than 8 bytes, this function will perform more work than necessary unless
+// the compiler is able to optimize the extra work away.
 #[inline]
-unsafe fn u8to64_le(buf: &[u8], start: usize, len: usize) -> u64 {
-    debug_assert!(len < 8);
-    let mut i = 0; // current byte index (from LSB) in the output u64
-    let mut out = 0;
-    if i + 3 < len {
-        out = load_int_le!(buf, start + i, u32) as u64;
+unsafe fn copy_nonoverlapping_small(src: *const u8, dst: *mut u8, count: usize) {
+    debug_assert!(count <= 8);
+
+    if count == 8 {
+        ptr::copy_nonoverlapping(src, dst, 8);
+        return;
+    }
+
+    let mut i = 0;
+    if i + 3 < count {
+        ptr::copy_nonoverlapping(src.add(i), dst.add(i), 4);
         i += 4;
     }
-    if i + 1 < len {
-        out |= (load_int_le!(buf, start + i, u16) as u64) << (i * 8);
+
+    if i + 1 < count {
+        ptr::copy_nonoverlapping(src.add(i), dst.add(i), 2);
         i += 2
     }
-    if i < len {
-        out |= (*buf.get_unchecked(start + i) as u64) << (i * 8);
+
+    if i < count {
+        *dst.add(i) = *src.add(i);
         i += 1;
     }
-    debug_assert_eq!(i, len);
-    out
+
+    debug_assert_eq!(i, count);
 }
 
+// # Implementation
+//
+// This implementation uses buffering to reduce the hashing cost for inputs
+// consisting of many small integers. Buffering simplifies the integration of
+// integer input--the integer write function typically just appends to the
+// buffer with a statically sized write, updates metadata, and returns.
+//
+// Buffering also prevents alternating between writes that do and do not trigger
+// the hashing process. Only when the entire buffer is full do we transition
+// into hashing. This allows us to keep the hash state in registers for longer,
+// instead of loading and storing it before and after processing each element.
+//
+// When a write fills the buffer, a buffer processing function is invoked to
+// hash all of the buffered input. The buffer processing functions are marked
+// `#[inline(never)]` so that they aren't inlined into the append functions,
+// which ensures the more frequently called append functions remain inlineable
+// and don't include register pushing/popping that would only be made necessary
+// by inclusion of the complex buffer processing path which uses those
+// registers.
+//
+// The buffer includes a "spill"--an extra element at the end--which simplifies
+// the integer write buffer processing path. The value that fills the buffer can
+// be written with a statically sized write that may spill over into the spill.
+// After the buffer is processed, the part of the value that spilled over can be
+// written from the spill to the beginning of the buffer with another statically
+// sized write. This write may copy more bytes than actually spilled over, but
+// we maintain the metadata such that any extra copied bytes will be ignored by
+// subsequent processing. Due to the static sizes, this scheme performs better
+// than copying the exact number of bytes needed into the end and beginning of
+// the buffer.
+//
+// The buffer is uninitialized, which improves performance, but may preclude
+// efficient implementation of alternative approaches. The improvement is not so
+// large that an alternative approach should be disregarded because it cannot be
+// efficiently implemented with an uninitialized buffer. On the other hand, an
+// uninitialized buffer may become more important should a larger one be used.
+//
+// # Platform Dependence
+//
+// The SipHash algorithm operates on byte sequences. It parses the input stream
+// as 8-byte little-endian integers. Therefore, given the same byte sequence, it
+// produces the same result on big- and little-endian hardware.
+//
+// However, the Hasher trait has methods which operate on multi-byte integers.
+// How they are converted into byte sequences can be endian-dependent (by using
+// native byte order) or independent (by consistently using either LE or BE byte
+// order). It can also be `isize` and `usize` size dependent (by using the
+// native size), or independent (by converting to a common size), supposing the
+// values can be represented in 32 bits.
+//
+// In order to make `SipHasher128` consistent with `SipHasher` in libstd, we
+// choose to do the integer to byte sequence conversion in the platform-
+// dependent way. Clients can achieve platform-independent hashing by widening
+// `isize` and `usize` integers to 64 bits on 32-bit systems and byte-swapping
+// integers on big-endian systems before passing them to the writing functions.
+// This causes the input byte sequence to look identical on big- and little-
+// endian systems (supposing `isize` and `usize` values can be represented in 32
+// bits), which ensures platform-independent results.
 impl SipHasher128 {
     #[inline]
     pub fn new_with_keys(key0: u64, key1: u64) -> SipHasher128 {
-        let mut state = SipHasher128 {
-            k0: key0,
-            k1: key1,
-            length: 0,
-            state: State { v0: 0, v1: 0, v2: 0, v3: 0 },
-            tail: 0,
-            ntail: 0,
+        let mut hasher = SipHasher128 {
+            nbuf: 0,
+            buf: MaybeUninit::uninit_array(),
+            state: State {
+                v0: key0 ^ 0x736f6d6570736575,
+                // The XOR with 0xee is only done on 128-bit algorithm version.
+                v1: key1 ^ (0x646f72616e646f6d ^ 0xee),
+                v2: key0 ^ 0x6c7967656e657261,
+                v3: key1 ^ 0x7465646279746573,
+            },
+            processed: 0,
         };
-        state.reset();
-        state
+
+        unsafe {
+            // Initialize spill because we read from it in `short_write_process_buffer`.
+            *hasher.buf.get_unchecked_mut(BUFFER_SPILL_INDEX) = MaybeUninit::zeroed();
+        }
+
+        hasher
     }
 
+    // A specialized write function for values with size <= 8.
     #[inline]
-    fn reset(&mut self) {
-        self.length = 0;
-        self.state.v0 = self.k0 ^ 0x736f6d6570736575;
-        self.state.v1 = self.k1 ^ 0x646f72616e646f6d;
-        self.state.v2 = self.k0 ^ 0x6c7967656e657261;
-        self.state.v3 = self.k1 ^ 0x7465646279746573;
-        self.ntail = 0;
-
-        // This is only done in the 128 bit version:
-        self.state.v1 ^= 0xee;
+    fn short_write<T>(&mut self, x: T) {
+        let size = mem::size_of::<T>();
+        let nbuf = self.nbuf;
+        debug_assert!(size <= 8);
+        debug_assert!(nbuf < BUFFER_SIZE);
+        debug_assert!(nbuf + size < BUFFER_WITH_SPILL_SIZE);
+
+        if nbuf + size < BUFFER_SIZE {
+            unsafe {
+                // The memcpy call is optimized away because the size is known.
+                let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf);
+                ptr::copy_nonoverlapping(&x as *const _ as *const u8, dst, size);
+            }
+
+            self.nbuf = nbuf + size;
+
+            return;
+        }
+
+        unsafe { self.short_write_process_buffer(x) }
     }
 
-    // A specialized write function for values with size <= 8.
-    //
-    // The input must be zero-extended to 64-bits by the caller. This extension
-    // isn't hashed, but the implementation requires it for correctness.
+    // A specialized write function for values with size <= 8 that should only
+    // be called when the write would cause the buffer to fill.
     //
-    // This function, given the same integer size and value, has the same effect
-    // on both little- and big-endian hardware. It operates on values without
-    // depending on their sequence in memory, so is independent of endianness.
-    //
-    // However, we want SipHasher128 to be platform-dependent, in order to be
-    // consistent with the platform-dependent SipHasher in libstd. In other
-    // words, we want:
-    //
-    // - little-endian: `write_u32(0xDDCCBBAA)` == `write([0xAA, 0xBB, 0xCC, 0xDD])`
-    // - big-endian:    `write_u32(0xDDCCBBAA)` == `write([0xDD, 0xCC, 0xBB, 0xAA])`
-    //
-    // Therefore, in order to produce endian-dependent results, SipHasher128's
-    // `write_xxx` Hasher trait methods byte-swap `x` prior to zero-extending.
-    //
-    // If clients of SipHasher128 itself want platform-independent results, they
-    // *also* must byte-swap integer inputs before invoking the `write_xxx`
-    // methods on big-endian hardware (that is, two byte-swaps must occur--one
-    // in the client, and one in SipHasher128). Additionally, they must extend
-    // `usize` and `isize` types to 64 bits on 32-bit systems.
-    #[inline]
-    fn short_write<T>(&mut self, _x: T, x: u64) {
+    // SAFETY: the write of `x` into `self.buf` starting at byte offset
+    // `self.nbuf` must cause `self.buf` to become fully initialized (and not
+    // overflow) if it wasn't already.
+    #[inline(never)]
+    unsafe fn short_write_process_buffer<T>(&mut self, x: T) {
         let size = mem::size_of::<T>();
-        self.length += size;
-
-        // The original number must be zero-extended, not sign-extended.
-        debug_assert!(if size < 8 { x >> (8 * size) == 0 } else { true });
-
-        // The number of bytes needed to fill `self.tail`.
-        let needed = 8 - self.ntail;
-
-        // SipHash parses the input stream as 8-byte little-endian integers.
-        // Inputs are put into `self.tail` until 8 bytes of data have been
-        // collected, and then that word is processed.
-        //
-        // For example, imagine that `self.tail` is 0x0000_00EE_DDCC_BBAA,
-        // `self.ntail` is 5 (because 5 bytes have been put into `self.tail`),
-        // and `needed` is therefore 3.
-        //
-        // - Scenario 1, `self.write_u8(0xFF)`: we have already zero-extended
-        //   the input to 0x0000_0000_0000_00FF. We now left-shift it five
-        //   bytes, giving 0x0000_FF00_0000_0000. We then bitwise-OR that value
-        //   into `self.tail`, resulting in 0x0000_FFEE_DDCC_BBAA.
-        //   (Zero-extension of the original input is critical in this scenario
-        //   because we don't want the high two bytes of `self.tail` to be
-        //   touched by the bitwise-OR.) `self.tail` is not yet full, so we
-        //   return early, after updating `self.ntail` to 6.
-        //
-        // - Scenario 2, `self.write_u32(0xIIHH_GGFF)`: we have already
-        //   zero-extended the input to 0x0000_0000_IIHH_GGFF. We now
-        //   left-shift it five bytes, giving 0xHHGG_FF00_0000_0000. We then
-        //   bitwise-OR that value into `self.tail`, resulting in
-        //   0xHHGG_FFEE_DDCC_BBAA. `self.tail` is now full, and we can use it
-        //   to update `self.state`. (As mentioned above, this assumes a
-        //   little-endian machine; on a big-endian machine we would have
-        //   byte-swapped 0xIIHH_GGFF in the caller, giving 0xFFGG_HHII, and we
-        //   would then end up bitwise-ORing 0xGGHH_II00_0000_0000 into
-        //   `self.tail`).
-        //
-        self.tail |= x << (8 * self.ntail);
-        if size < needed {
-            self.ntail += size;
+        let nbuf = self.nbuf;
+        debug_assert!(size <= 8);
+        debug_assert!(nbuf < BUFFER_SIZE);
+        debug_assert!(nbuf + size >= BUFFER_SIZE);
+        debug_assert!(nbuf + size < BUFFER_WITH_SPILL_SIZE);
+
+        // Copy first part of input into end of buffer, possibly into spill
+        // element. The memcpy call is optimized away because the size is known.
+        let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf);
+        ptr::copy_nonoverlapping(&x as *const _ as *const u8, dst, size);
+
+        // Process buffer.
+        for i in 0..BUFFER_CAPACITY {
+            let elem = self.buf.get_unchecked(i).assume_init().to_le();
+            self.state.v3 ^= elem;
+            Sip24Rounds::c_rounds(&mut self.state);
+            self.state.v0 ^= elem;
+        }
+
+        // Copy remaining input into start of buffer by copying size - 1
+        // elements from spill (at most size - 1 bytes could have overflowed
+        // into the spill). The memcpy call is optimized away because the size
+        // is known. And the whole copy is optimized away for size == 1.
+        let src = self.buf.get_unchecked(BUFFER_SPILL_INDEX) as *const _ as *const u8;
+        ptr::copy_nonoverlapping(src, self.buf.as_mut_ptr() as *mut u8, size - 1);
+
+        // This function should only be called when the write fills the buffer.
+        // Therefore, when size == 1, the new `self.nbuf` must be zero. The size
+        // is statically known, so the branch is optimized away.
+        self.nbuf = if size == 1 { 0 } else { nbuf + size - BUFFER_SIZE };
+        self.processed += BUFFER_SIZE;
+    }
+
+    // A write function for byte slices.
+    #[inline]
+    fn slice_write(&mut self, msg: &[u8]) {
+        let length = msg.len();
+        let nbuf = self.nbuf;
+        debug_assert!(nbuf < BUFFER_SIZE);
+
+        if nbuf + length < BUFFER_SIZE {
+            unsafe {
+                let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf);
+
+                if length <= 8 {
+                    copy_nonoverlapping_small(msg.as_ptr(), dst, length);
+                } else {
+                    // This memcpy is *not* optimized away.
+                    ptr::copy_nonoverlapping(msg.as_ptr(), dst, length);
+                }
+            }
+
+            self.nbuf = nbuf + length;
+
             return;
         }
 
-        // `self.tail` is full, process it.
-        self.state.v3 ^= self.tail;
-        Sip24Rounds::c_rounds(&mut self.state);
-        self.state.v0 ^= self.tail;
-
-        // Continuing scenario 2: we have one byte left over from the input. We
-        // set `self.ntail` to 1 and `self.tail` to `0x0000_0000_IIHH_GGFF >>
-        // 8*3`, which is 0x0000_0000_0000_00II. (Or on a big-endian machine
-        // the prior byte-swapping would leave us with 0x0000_0000_0000_00FF.)
-        //
-        // The `if` is needed to avoid shifting by 64 bits, which Rust
-        // complains about.
-        self.ntail = size - needed;
-        self.tail = if needed < 8 { x >> (8 * needed) } else { 0 };
+        unsafe { self.slice_write_process_buffer(msg) }
+    }
+
+    // A write function for byte slices that should only be called when the
+    // write would cause the buffer to fill.
+    //
+    // SAFETY: `self.buf` must be initialized up to the byte offset `self.nbuf`,
+    // and `msg` must contain enough bytes to initialize the rest of the element
+    // containing the byte offset `self.nbuf`.
+    #[inline(never)]
+    unsafe fn slice_write_process_buffer(&mut self, msg: &[u8]) {
+        let length = msg.len();
+        let nbuf = self.nbuf;
+        debug_assert!(nbuf < BUFFER_SIZE);
+        debug_assert!(nbuf + length >= BUFFER_SIZE);
+
+        // Always copy first part of input into current element of buffer.
+        // This function should only be called when the write fills the buffer,
+        // so we know that there is enough input to fill the current element.
+        let valid_in_elem = nbuf % ELEM_SIZE;
+        let needed_in_elem = ELEM_SIZE - valid_in_elem;
+
+        let src = msg.as_ptr();
+        let dst = (self.buf.as_mut_ptr() as *mut u8).add(nbuf);
+        copy_nonoverlapping_small(src, dst, needed_in_elem);
+
+        // Process buffer.
+
+        // Using `nbuf / ELEM_SIZE + 1` rather than `(nbuf + needed_in_elem) /
+        // ELEM_SIZE` to show the compiler that this loop's upper bound is > 0.
+        // We know that is true, because last step ensured we have a full
+        // element in the buffer.
+        let last = nbuf / ELEM_SIZE + 1;
+
+        for i in 0..last {
+            let elem = self.buf.get_unchecked(i).assume_init().to_le();
+            self.state.v3 ^= elem;
+            Sip24Rounds::c_rounds(&mut self.state);
+            self.state.v0 ^= elem;
+        }
+
+        // Process the remaining element-sized chunks of input.
+        let mut processed = needed_in_elem;
+        let input_left = length - processed;
+        let elems_left = input_left / ELEM_SIZE;
+        let extra_bytes_left = input_left % ELEM_SIZE;
+
+        for _ in 0..elems_left {
+            let elem = (msg.as_ptr().add(processed) as *const u64).read_unaligned().to_le();
+            self.state.v3 ^= elem;
+            Sip24Rounds::c_rounds(&mut self.state);
+            self.state.v0 ^= elem;
+            processed += ELEM_SIZE;
+        }
+
+        // Copy remaining input into start of buffer.
+        let src = msg.as_ptr().add(processed);
+        let dst = self.buf.as_mut_ptr() as *mut u8;
+        copy_nonoverlapping_small(src, dst, extra_bytes_left);
+
+        self.nbuf = extra_bytes_left;
+        self.processed += nbuf + processed;
     }
 
     #[inline]
     pub fn finish128(mut self) -> (u64, u64) {
-        let b: u64 = ((self.length as u64 & 0xff) << 56) | self.tail;
+        debug_assert!(self.nbuf < BUFFER_SIZE);
 
-        self.state.v3 ^= b;
-        Sip24Rounds::c_rounds(&mut self.state);
-        self.state.v0 ^= b;
+        // Process full elements in buffer.
+        let last = self.nbuf / ELEM_SIZE;
 
-        self.state.v2 ^= 0xee;
-        Sip24Rounds::d_rounds(&mut self.state);
-        let _0 = self.state.v0 ^ self.state.v1 ^ self.state.v2 ^ self.state.v3;
+        // Since we're consuming self, avoid updating members for a potential
+        // performance gain.
+        let mut state = self.state;
+
+        for i in 0..last {
+            let elem = unsafe { self.buf.get_unchecked(i).assume_init().to_le() };
+            state.v3 ^= elem;
+            Sip24Rounds::c_rounds(&mut state);
+            state.v0 ^= elem;
+        }
+
+        // Get remaining partial element.
+        let elem = if self.nbuf % ELEM_SIZE != 0 {
+            unsafe {
+                // Ensure element is initialized by writing zero bytes. At most
+                // `ELEM_SIZE - 1` are required given the above check. It's safe
+                // to write this many because we have the spill and we maintain
+                // `self.nbuf` such that this write will start before the spill.
+                let dst = (self.buf.as_mut_ptr() as *mut u8).add(self.nbuf);
+                ptr::write_bytes(dst, 0, ELEM_SIZE - 1);
+                self.buf.get_unchecked(last).assume_init().to_le()
+            }
+        } else {
+            0
+        };
+
+        // Finalize the hash.
+        let length = self.processed + self.nbuf;
+        let b: u64 = ((length as u64 & 0xff) << 56) | elem;
+
+        state.v3 ^= b;
+        Sip24Rounds::c_rounds(&mut state);
+        state.v0 ^= b;
+
+        state.v2 ^= 0xee;
+        Sip24Rounds::d_rounds(&mut state);
+        let _0 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;
+
+        state.v1 ^= 0xdd;
+        Sip24Rounds::d_rounds(&mut state);
+        let _1 = state.v0 ^ state.v1 ^ state.v2 ^ state.v3;
 
-        self.state.v1 ^= 0xdd;
-        Sip24Rounds::d_rounds(&mut self.state);
-        let _1 = self.state.v0 ^ self.state.v1 ^ self.state.v2 ^ self.state.v3;
         (_0, _1)
     }
 }
@@ -230,92 +412,57 @@ impl SipHasher128 {
 impl Hasher for SipHasher128 {
     #[inline]
     fn write_u8(&mut self, i: u8) {
-        self.short_write(i, i as u64);
+        self.short_write(i);
     }
 
     #[inline]
     fn write_u16(&mut self, i: u16) {
-        self.short_write(i, i.to_le() as u64);
+        self.short_write(i);
     }
 
     #[inline]
     fn write_u32(&mut self, i: u32) {
-        self.short_write(i, i.to_le() as u64);
+        self.short_write(i);
     }
 
     #[inline]
     fn write_u64(&mut self, i: u64) {
-        self.short_write(i, i.to_le() as u64);
+        self.short_write(i);
     }
 
     #[inline]
     fn write_usize(&mut self, i: usize) {
-        self.short_write(i, i.to_le() as u64);
+        self.short_write(i);
     }
 
     #[inline]
     fn write_i8(&mut self, i: i8) {
-        self.short_write(i, i as u8 as u64);
+        self.short_write(i as u8);
     }
 
     #[inline]
     fn write_i16(&mut self, i: i16) {
-        self.short_write(i, (i as u16).to_le() as u64);
+        self.short_write(i as u16);
     }
 
     #[inline]
     fn write_i32(&mut self, i: i32) {
-        self.short_write(i, (i as u32).to_le() as u64);
+        self.short_write(i as u32);
     }
 
     #[inline]
     fn write_i64(&mut self, i: i64) {
-        self.short_write(i, (i as u64).to_le() as u64);
+        self.short_write(i as u64);
     }
 
     #[inline]
     fn write_isize(&mut self, i: isize) {
-        self.short_write(i, (i as usize).to_le() as u64);
+        self.short_write(i as usize);
     }
 
     #[inline]
     fn write(&mut self, msg: &[u8]) {
-        let length = msg.len();
-        self.length += length;
-
-        let mut needed = 0;
-
-        if self.ntail != 0 {
-            needed = 8 - self.ntail;
-            self.tail |= unsafe { u8to64_le(msg, 0, cmp::min(length, needed)) } << (8 * self.ntail);
-            if length < needed {
-                self.ntail += length;
-                return;
-            } else {
-                self.state.v3 ^= self.tail;
-                Sip24Rounds::c_rounds(&mut self.state);
-                self.state.v0 ^= self.tail;
-                self.ntail = 0;
-            }
-        }
-
-        // Buffered tail is now flushed, process new input.
-        let len = length - needed;
-        let left = len & 0x7;
-
-        let mut i = needed;
-        while i < len - left {
-            let mi = unsafe { load_int_le!(msg, i, u64) };
-
-            self.state.v3 ^= mi;
-            Sip24Rounds::c_rounds(&mut self.state);
-            self.state.v0 ^= mi;
-
-            i += 8;
-        }
-
-        self.tail = unsafe { u8to64_le(msg, i, left) };
-        self.ntail = left;
+        self.slice_write(msg);
     }
 
     fn finish(&self) -> u64 {
diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs
index 2e2274a7b77..5fe967c4158 100644
--- a/compiler/rustc_data_structures/src/sip128/tests.rs
+++ b/compiler/rustc_data_structures/src/sip128/tests.rs
@@ -450,3 +450,48 @@ fn test_short_write_works() {
 
     assert_eq!(h1_hash, h2_hash);
 }
+
+macro_rules! test_fill_buffer {
+    ($type:ty, $write_method:ident) => {{
+        // Test filling and overfilling the buffer from all possible offsets
+        // for a given integer type and its corresponding write method.
+        const SIZE: usize = std::mem::size_of::<$type>();
+        let input = [42; BUFFER_SIZE];
+        let x = 0x01234567_89ABCDEF_76543210_FEDCBA98_u128 as $type;
+        let x_bytes = &x.to_ne_bytes();
+
+        for i in 1..=SIZE {
+            let s = &input[..BUFFER_SIZE - i];
+
+            let mut h1 = SipHasher128::new_with_keys(7, 13);
+            h1.write(s);
+            h1.$write_method(x);
+
+            let mut h2 = SipHasher128::new_with_keys(7, 13);
+            h2.write(s);
+            h2.write(x_bytes);
+
+            let h1_hash = h1.finish128();
+            let h2_hash = h2.finish128();
+
+            assert_eq!(h1_hash, h2_hash);
+        }
+    }};
+}
+
+#[test]
+fn test_fill_buffer() {
+    test_fill_buffer!(u8, write_u8);
+    test_fill_buffer!(u16, write_u16);
+    test_fill_buffer!(u32, write_u32);
+    test_fill_buffer!(u64, write_u64);
+    test_fill_buffer!(u128, write_u128);
+    test_fill_buffer!(usize, write_usize);
+
+    test_fill_buffer!(i8, write_i8);
+    test_fill_buffer!(i16, write_i16);
+    test_fill_buffer!(i32, write_i32);
+    test_fill_buffer!(i64, write_i64);
+    test_fill_buffer!(i128, write_i128);
+    test_fill_buffer!(isize, write_isize);
+}