inliner: Avoid query cycles when optimizing generators

The HIR Id trick is insufficient to prevent query cycles when optimizing
generators, since merely requesting a layout of a generator also
computes its `optimized_mir`.

Make no attempts to inline functions into generators within the same
crate to avoid query cycles.

2 files changed, 26 insertions, 2 deletions

diff --git a/compiler/rustc_mir/src/transform/inline.rs b/compiler/rustc_mir/src/transform/inline.rs
index 315d4fa9d47..428f4e138c7 100644
--- a/compiler/rustc_mir/src/transform/inline.rs
+++ b/compiler/rustc_mir/src/transform/inline.rs
@@ -107,8 +107,14 @@ impl Inliner<'tcx> {
                     // Avoid a cycle here by only using `optimized_mir` only if we have
                     // a lower `HirId` than the callee. This ensures that the callee will
                     // not inline us. This trick only works without incremental compilation.
-                    // So don't do it if that is enabled.
-                    if !self.tcx.dep_graph.is_fully_enabled() && self_hir_id < callee_hir_id {
+                    // So don't do it if that is enabled. Also avoid inlining into generators,
+                    // since their `optimized_mir` is used for layout computation, which can
+                    // create a cycle, even when no attempt is made to inline the function
+                    // in the other direction.
+                    if !self.tcx.dep_graph.is_fully_enabled()
+                        && self_hir_id < callee_hir_id
+                        && caller_body.generator_kind.is_none()
+                    {
                         self.tcx.optimized_mir(callsite.callee)
                     } else {
                         continue;
diff --git a/src/test/mir-opt/inline/inline-async.rs b/src/test/mir-opt/inline/inline-async.rs
new file mode 100644
index 00000000000..5c838159b98
--- /dev/null
+++ b/src/test/mir-opt/inline/inline-async.rs
@@ -0,0 +1,18 @@
+// Checks that inliner doesn't introduce cycles when optimizing generators.
+// The outcome of optimization is not verfied, just the absence of the cycle.
+// Regression test for #76181.
+//
+// edition:2018
+
+#![crate_type = "lib"]
+
+pub struct S;
+
+impl S {
+    pub async fn g(&mut self) {
+        self.h();
+    }
+    pub fn h(&mut self) {
+        let _ = self.g();
+    }
+}