Allocate candidate vectors as we sort them

7 files changed, 56 insertions, 88 deletions

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index aea52fc497f..a1f620e7baf 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1653,10 +1653,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &'b mut [&'c mut Candidate<'pat, 'tcx>],
         FxIndexMap<TestBranch<'tcx>, Vec<&'b mut Candidate<'pat, 'tcx>>>,
     ) {
-        // For each of the N possible outcomes, create a (initially empty) vector of candidates.
-        // Those are the candidates that apply if the test has that particular outcome.
-        let mut target_candidates: FxIndexMap<_, Vec<&mut Candidate<'pat, 'tcx>>> =
-            test.targets().into_iter().map(|branch| (branch, Vec::new())).collect();
+        // For each of the possible outcomes, collect vector of candidates that apply if the test
+        // has that particular outcome.
+        let mut target_candidates: FxIndexMap<_, Vec<&mut Candidate<'_, '_>>> = Default::default();
 
         let total_candidate_count = candidates.len();
 
@@ -1668,7 +1667,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 break;
             };
             let (candidate, rest) = candidates.split_first_mut().unwrap();
-            target_candidates[&branch].push(candidate);
+            target_candidates.entry(branch).or_insert_with(Vec::new).push(candidate);
             candidates = rest;
         }
 
@@ -1809,31 +1808,32 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             otherwise_block
         };
 
-        // For each outcome of test, process the candidates that still
-        // apply. Collect a list of blocks where control flow will
-        // branch if one of the `target_candidate` sets is not
-        // exhaustive.
+        // For each outcome of test, process the candidates that still apply.
         let target_blocks: FxIndexMap<_, _> = target_candidates
             .into_iter()
             .map(|(branch, mut candidates)| {
-                if !candidates.is_empty() {
-                    let candidate_start = self.cfg.start_new_block();
-                    self.match_candidates(
-                        span,
-                        scrutinee_span,
-                        candidate_start,
-                        remainder_start,
-                        &mut *candidates,
-                    );
-                    (branch, candidate_start)
-                } else {
-                    (branch, remainder_start)
-                }
+                let candidate_start = self.cfg.start_new_block();
+                self.match_candidates(
+                    span,
+                    scrutinee_span,
+                    candidate_start,
+                    remainder_start,
+                    &mut *candidates,
+                );
+                (branch, candidate_start)
             })
             .collect();
 
         // Perform the test, branching to one of N blocks.
-        self.perform_test(span, scrutinee_span, start_block, &match_place, &test, target_blocks);
+        self.perform_test(
+            span,
+            scrutinee_span,
+            start_block,
+            remainder_start,
+            &match_place,
+            &test,
+            target_blocks,
+        );
     }
 
     /// Determine the fake borrows that are needed from a set of places that
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index d003ae8d803..4244eb45045 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -127,6 +127,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_start_span: Span,
         scrutinee_span: Span,
         block: BasicBlock,
+        otherwise_block: BasicBlock,
         place_builder: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         target_blocks: FxIndexMap<TestBranch<'tcx>, BasicBlock>,
@@ -134,14 +135,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let place = place_builder.to_place(self);
         let place_ty = place.ty(&self.local_decls, self.tcx);
         debug!(?place, ?place_ty);
-        let target_block = |branch| target_blocks[&branch];
+        let target_block = |branch| target_blocks.get(&branch).copied().unwrap_or(otherwise_block);
 
         let source_info = self.source_info(test.span);
         match test.kind {
             TestKind::Switch { adt_def, ref variants } => {
                 // Variants is a BitVec of indexes into adt_def.variants.
                 let num_enum_variants = adt_def.variants().len();
-                debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
                 let otherwise_block = target_block(TestBranch::Failure);
                 let tcx = self.tcx;
                 let switch_targets = SwitchTargets::new(
@@ -185,7 +185,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             TestKind::SwitchInt { ref options } => {
                 // The switch may be inexhaustive so we have a catch-all block
-                debug_assert_eq!(options.len() + 1, target_blocks.len());
                 let otherwise_block = target_block(TestBranch::Failure);
                 let switch_targets = SwitchTargets::new(
                     options
@@ -201,7 +200,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::If => {
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 let terminator =
@@ -211,7 +209,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             TestKind::Eq { value, ty } => {
                 let tcx = self.tcx;
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 if let ty::Adt(def, _) = ty.kind()
@@ -290,7 +287,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::Range(ref range) => {
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success = target_block(TestBranch::Success);
                 let fail = target_block(TestBranch::Failure);
                 // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
@@ -337,7 +333,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // expected = <N>
                 let expected = self.push_usize(block, source_info, len);
 
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 // result = actual == expected OR result = actual < expected
@@ -753,33 +748,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-impl<'tcx> Test<'tcx> {
-    pub(super) fn targets(&self) -> Vec<TestBranch<'tcx>> {
-        match self.kind {
-            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } | TestKind::If => {
-                vec![TestBranch::Success, TestBranch::Failure]
-            }
-            TestKind::Switch { adt_def, .. } => {
-                // While the switch that we generate doesn't test for all
-                // variants, we have a target for each variant and the
-                // otherwise case, and we make sure that all of the cases not
-                // specified have the same block.
-                adt_def
-                    .variants()
-                    .indices()
-                    .map(|idx| TestBranch::Variant(idx))
-                    .chain([TestBranch::Failure])
-                    .collect()
-            }
-            TestKind::SwitchInt { ref options } => options
-                .iter()
-                .map(|(val, bits)| TestBranch::Constant(*val, *bits))
-                .chain([TestBranch::Failure])
-                .collect(),
-        }
-    }
-}
-
 fn is_switch_ty(ty: Ty<'_>) -> bool {
     ty.is_integral() || ty.is_char()
 }
diff --git a/tests/mir-opt/building/issue_49232.main.built.after.mir b/tests/mir-opt/building/issue_49232.main.built.after.mir
index 166e28ce51d..d09a1748a8b 100644
--- a/tests/mir-opt/building/issue_49232.main.built.after.mir
+++ b/tests/mir-opt/building/issue_49232.main.built.after.mir
@@ -25,7 +25,7 @@ fn main() -> () {
         StorageLive(_3);
         _3 = const true;
         PlaceMention(_3);
-        switchInt(_3) -> [0: bb6, otherwise: bb4];
+        switchInt(_3) -> [0: bb4, otherwise: bb6];
     }
 
     bb3: {
@@ -34,8 +34,7 @@ fn main() -> () {
     }
 
     bb4: {
-        _0 = const ();
-        goto -> bb13;
+        falseEdge -> [real: bb8, imaginary: bb6];
     }
 
     bb5: {
@@ -43,7 +42,8 @@ fn main() -> () {
     }
 
     bb6: {
-        falseEdge -> [real: bb8, imaginary: bb4];
+        _0 = const ();
+        goto -> bb13;
     }
 
     bb7: {
diff --git a/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir b/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
index 4e91eb6f76f..194afdf7dd8 100644
--- a/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
+++ b/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
@@ -28,7 +28,7 @@ fn full_tested_match() -> () {
         _2 = Option::<i32>::Some(const 42_i32);
         PlaceMention(_2);
         _3 = discriminant(_2);
-        switchInt(move _3) -> [0: bb2, 1: bb4, otherwise: bb1];
+        switchInt(move _3) -> [0: bb5, 1: bb2, otherwise: bb1];
     }
 
     bb1: {
@@ -37,20 +37,20 @@ fn full_tested_match() -> () {
     }
 
     bb2: {
-        _1 = (const 3_i32, const 3_i32);
-        goto -> bb13;
+        falseEdge -> [real: bb7, imaginary: bb3];
     }
 
     bb3: {
-        goto -> bb1;
+        falseEdge -> [real: bb12, imaginary: bb5];
     }
 
     bb4: {
-        falseEdge -> [real: bb7, imaginary: bb5];
+        goto -> bb1;
     }
 
     bb5: {
-        falseEdge -> [real: bb12, imaginary: bb2];
+        _1 = (const 3_i32, const 3_i32);
+        goto -> bb13;
     }
 
     bb6: {
@@ -91,7 +91,7 @@ fn full_tested_match() -> () {
     bb11: {
         StorageDead(_7);
         StorageDead(_6);
-        goto -> bb5;
+        goto -> bb3;
     }
 
     bb12: {
diff --git a/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir b/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
index 0c67cc9f71e..ae83075434f 100644
--- a/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
+++ b/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
@@ -28,7 +28,7 @@ fn full_tested_match2() -> () {
         _2 = Option::<i32>::Some(const 42_i32);
         PlaceMention(_2);
         _3 = discriminant(_2);
-        switchInt(move _3) -> [0: bb2, 1: bb4, otherwise: bb1];
+        switchInt(move _3) -> [0: bb5, 1: bb2, otherwise: bb1];
     }
 
     bb1: {
@@ -37,18 +37,10 @@ fn full_tested_match2() -> () {
     }
 
     bb2: {
-        falseEdge -> [real: bb12, imaginary: bb5];
+        falseEdge -> [real: bb7, imaginary: bb5];
     }
 
     bb3: {
-        goto -> bb1;
-    }
-
-    bb4: {
-        falseEdge -> [real: bb7, imaginary: bb2];
-    }
-
-    bb5: {
         StorageLive(_9);
         _9 = ((_2 as Some).0: i32);
         StorageLive(_10);
@@ -59,6 +51,14 @@ fn full_tested_match2() -> () {
         goto -> bb13;
     }
 
+    bb4: {
+        goto -> bb1;
+    }
+
+    bb5: {
+        falseEdge -> [real: bb12, imaginary: bb3];
+    }
+
     bb6: {
         goto -> bb1;
     }
@@ -97,7 +97,7 @@ fn full_tested_match2() -> () {
     bb11: {
         StorageDead(_7);
         StorageDead(_6);
-        falseEdge -> [real: bb5, imaginary: bb2];
+        falseEdge -> [real: bb3, imaginary: bb5];
     }
 
     bb12: {
diff --git a/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
index 32a8dd8b8b4..0aed59be794 100644
--- a/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
@@ -30,7 +30,7 @@
           StorageDead(_5);
           StorageDead(_4);
           _8 = discriminant((_3.0: std::option::Option<u32>));
--         switchInt(move _8) -> [0: bb2, 1: bb3, otherwise: bb1];
+-         switchInt(move _8) -> [0: bb3, 1: bb2, otherwise: bb1];
 +         StorageLive(_11);
 +         _11 = discriminant((_3.1: std::option::Option<u32>));
 +         StorageLive(_12);
@@ -48,12 +48,12 @@
   
       bb2: {
 -         _6 = discriminant((_3.1: std::option::Option<u32>));
--         switchInt(move _6) -> [0: bb5, otherwise: bb1];
+-         switchInt(move _6) -> [1: bb4, otherwise: bb1];
 -     }
 - 
 -     bb3: {
 -         _7 = discriminant((_3.1: std::option::Option<u32>));
--         switchInt(move _7) -> [1: bb4, otherwise: bb1];
+-         switchInt(move _7) -> [0: bb5, otherwise: bb1];
 -     }
 - 
 -     bb4: {
diff --git a/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
index d7908ab3cd2..d446a5003a3 100644
--- a/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
@@ -36,7 +36,7 @@
           StorageDead(_5);
           StorageDead(_4);
           _8 = discriminant((_3.0: std::option::Option<u32>));
-          switchInt(move _8) -> [0: bb2, 1: bb4, otherwise: bb1];
+          switchInt(move _8) -> [0: bb3, 1: bb2, otherwise: bb1];
       }
   
       bb1: {
@@ -45,17 +45,17 @@
   
       bb2: {
           _6 = discriminant((_3.1: std::option::Option<u32>));
-          switchInt(move _6) -> [0: bb3, 1: bb7, otherwise: bb1];
+          switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1];
       }
   
       bb3: {
-          _0 = const 3_u32;
-          goto -> bb8;
+          _7 = discriminant((_3.1: std::option::Option<u32>));
+          switchInt(move _7) -> [0: bb4, 1: bb7, otherwise: bb1];
       }
   
       bb4: {
-          _7 = discriminant((_3.1: std::option::Option<u32>));
-          switchInt(move _7) -> [0: bb6, 1: bb5, otherwise: bb1];
+          _0 = const 3_u32;
+          goto -> bb8;
       }
   
       bb5: {