about summary refs log tree commit diff
path: root/compiler
diff options
context:
space:
mode:
Diffstat (limited to 'compiler')
-rw-r--r--compiler/rustc_interface/src/tests.rs1
-rw-r--r--compiler/rustc_metadata/src/rmeta/decoder.rs4
-rw-r--r--compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs1
-rw-r--r--compiler/rustc_metadata/src/rmeta/encoder.rs5
-rw-r--r--compiler/rustc_metadata/src/rmeta/mod.rs1
-rw-r--r--compiler/rustc_metadata/src/rmeta/table.rs24
-rw-r--r--compiler/rustc_middle/src/middle/codegen_fn_attrs.rs8
-rw-r--r--compiler/rustc_middle/src/query/mod.rs5
-rw-r--r--compiler/rustc_middle/src/ty/instance.rs9
-rw-r--r--compiler/rustc_mir_transform/src/cross_crate_inline.rs119
-rw-r--r--compiler/rustc_mir_transform/src/inline.rs14
-rw-r--r--compiler/rustc_mir_transform/src/lib.rs2
-rw-r--r--compiler/rustc_passes/src/reachable.rs58
-rw-r--r--compiler/rustc_session/src/options.rs2
14 files changed, 184 insertions, 69 deletions
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 7799af37008..ec4fd78994e 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -770,6 +770,7 @@ fn test_unstable_options_tracking_hash() {
     );
     tracked!(codegen_backend, Some("abc".to_string()));
     tracked!(crate_attr, vec!["abc".to_string()]);
+    tracked!(cross_crate_inline_threshold, Some(200));
     tracked!(debug_info_for_profiling, true);
     tracked!(debug_macros, true);
     tracked!(dep_info_omit_d_target, true);
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index 846f8d25025..d6ceaa8a091 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -1273,6 +1273,10 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.root.tables.optimized_mir.get(self, id).is_some()
     }
 
+    fn cross_crate_inlinable(self, id: DefIndex) -> bool {
+        self.root.tables.cross_crate_inlinable.get(self, id).unwrap_or(false)
+    }
+
     fn get_fn_has_self_parameter(self, id: DefIndex, sess: &'a Session) -> bool {
         self.root
             .tables
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index f27eee0d79a..6b6c0d52742 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -287,6 +287,7 @@ provide! { tcx, def_id, other, cdata,
     item_attrs => { tcx.arena.alloc_from_iter(cdata.get_item_attrs(def_id.index, tcx.sess)) }
     is_mir_available => { cdata.is_item_mir_available(def_id.index) }
     is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) }
+    cross_crate_inlinable => { cdata.cross_crate_inlinable(def_id.index) }
 
     dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) }
     is_private_dep => {
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index dee2326ae32..7c406f9bdda 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -1046,7 +1046,7 @@ fn should_encode_mir(
                 || (tcx.sess.opts.output_types.should_codegen()
                     && reachable_set.contains(&def_id)
                     && (generics.requires_monomorphization(tcx)
-                        || tcx.codegen_fn_attrs(def_id).requests_inline()));
+                        || tcx.cross_crate_inlinable(def_id)));
             // The function has a `const` modifier or is in a `#[const_trait]`.
             let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id())
                 || tcx.is_const_default_method(def_id.to_def_id());
@@ -1615,6 +1615,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             debug!("EntryBuilder::encode_mir({:?})", def_id);
             if encode_opt {
                 record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id));
+                self.tables
+                    .cross_crate_inlinable
+                    .set(def_id.to_def_id().index, Some(self.tcx.cross_crate_inlinable(def_id)));
                 record!(self.tables.closure_saved_names_of_captured_variables[def_id.to_def_id()]
                     <- tcx.closure_saved_names_of_captured_variables(def_id));
 
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index 42764af52c4..2609767a85c 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -427,6 +427,7 @@ define_tables! {
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
     optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
+    cross_crate_inlinable: Table<DefIndex, bool>,
     closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
     mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index bb1320942b0..34118e9e8a3 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -299,6 +299,30 @@ impl FixedSizeEncoding for bool {
     }
 }
 
+impl FixedSizeEncoding for Option<bool> {
+    type ByteArray = [u8; 1];
+
+    #[inline]
+    fn from_bytes(b: &[u8; 1]) -> Self {
+        match b[0] {
+            0 => Some(false),
+            1 => Some(true),
+            2 => None,
+            _ => unreachable!(),
+        }
+    }
+
+    #[inline]
+    fn write_to_bytes(self, b: &mut [u8; 1]) {
+        debug_assert!(!self.is_default());
+        b[0] = match self {
+            Some(false) => 0,
+            Some(true) => 1,
+            None => 2,
+        };
+    }
+}
+
 impl FixedSizeEncoding for UnusedGenericParams {
     type ByteArray = [u8; 4];
 
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 4e5725876c4..f758c1d5e6f 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -126,14 +126,6 @@ impl CodegenFnAttrs {
         }
     }
 
-    /// Returns `true` if `#[inline]` or `#[inline(always)]` is present.
-    pub fn requests_inline(&self) -> bool {
-        match self.inline {
-            InlineAttr::Hint | InlineAttr::Always => true,
-            InlineAttr::None | InlineAttr::Never => false,
-        }
-    }
-
     /// Returns `true` if it looks like this symbol needs to be exported, for example:
     ///
     /// * `#[no_mangle]` is present
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 340c5a769db..940ab190ffc 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -2202,6 +2202,11 @@ rustc_queries! {
     query generics_require_sized_self(def_id: DefId) -> bool {
         desc { "check whether the item has a `where Self: Sized` bound" }
     }
+
+    query cross_crate_inlinable(def_id: DefId) -> bool {
+        desc { "whether the item should be made inlinable across crates" }
+        separate_provide_extern
+    }
 }
 
 rustc_query_append! { define_callbacks! }
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 0a425be52ff..0b308d5dec1 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -245,16 +245,15 @@ impl<'tcx> InstanceDef<'tcx> {
             // drops of `Option::None` before LTO. We also respect the intent of
             // `#[inline]` on `Drop::drop` implementations.
             return ty.ty_adt_def().map_or(true, |adt_def| {
-                adt_def.destructor(tcx).map_or_else(
-                    || adt_def.is_enum(),
-                    |dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
-                )
+                adt_def
+                    .destructor(tcx)
+                    .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
             });
         }
         if let ty::InstanceDef::ThreadLocalShim(..) = *self {
             return false;
         }
-        tcx.codegen_fn_attrs(self.def_id()).requests_inline()
+        tcx.cross_crate_inlinable(self.def_id())
     }
 
     pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {
diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
new file mode 100644
index 00000000000..24d081f2ac9
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -0,0 +1,119 @@
+use rustc_attr::InlineAttr;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::*;
+use rustc_middle::query::Providers;
+use rustc_middle::ty::TyCtxt;
+use rustc_session::config::OptLevel;
+
+pub fn provide(providers: &mut Providers) {
+    providers.cross_crate_inlinable = cross_crate_inlinable;
+}
+
+fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+    let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
+    // If this has an extern indicator, then this function is globally shared and thus will not
+    // generate cgu-internal copies which would make it cross-crate inlinable.
+    if codegen_fn_attrs.contains_extern_indicator() {
+        return false;
+    }
+
+    // Obey source annotations first; this is important because it means we can use
+    // #[inline(never)] to force code generation.
+    match codegen_fn_attrs.inline {
+        InlineAttr::Never => return false,
+        InlineAttr::Hint | InlineAttr::Always => return true,
+        _ => {}
+    }
+
+    // This just reproduces the logic from Instance::requires_inline.
+    match tcx.def_kind(def_id) {
+        DefKind::Ctor(..) | DefKind::Closure => return true,
+        DefKind::Fn | DefKind::AssocFn => {}
+        _ => return false,
+    }
+
+    // Don't do any inference when incremental compilation is enabled; the additional inlining that
+    // inference permits also creates more work for small edits.
+    if tcx.sess.opts.incremental.is_some() {
+        return false;
+    }
+
+    // Don't do any inference unless optimizations are enabled.
+    if matches!(tcx.sess.opts.optimize, OptLevel::No) {
+        return false;
+    }
+
+    if !tcx.is_mir_available(def_id) {
+        return false;
+    }
+
+    let mir = tcx.optimized_mir(def_id);
+    let mut checker =
+        CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
+    checker.visit_body(mir);
+    checker.calls == 0
+        && checker.resumes == 0
+        && checker.landing_pads == 0
+        && checker.statements
+            <= tcx.sess.opts.unstable_opts.cross_crate_inline_threshold.unwrap_or(100)
+}
+
+struct CostChecker<'b, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    callee_body: &'b Body<'tcx>,
+    calls: usize,
+    statements: usize,
+    landing_pads: usize,
+    resumes: usize,
+}
+
+impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+        // Don't count StorageLive/StorageDead in the inlining cost.
+        match statement.kind {
+            StatementKind::StorageLive(_)
+            | StatementKind::StorageDead(_)
+            | StatementKind::Deinit(_)
+            | StatementKind::Nop => {}
+            _ => self.statements += 1,
+        }
+    }
+
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
+        let tcx = self.tcx;
+        match terminator.kind {
+            TerminatorKind::Drop { ref place, unwind, .. } => {
+                let ty = place.ty(self.callee_body, tcx).ty;
+                if !ty.is_trivially_pure_clone_copy() {
+                    self.calls += 1;
+                    if let UnwindAction::Cleanup(_) = unwind {
+                        self.landing_pads += 1;
+                    }
+                }
+            }
+            TerminatorKind::Call { unwind, .. } => {
+                self.calls += 1;
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.landing_pads += 1;
+                }
+            }
+            TerminatorKind::Assert { unwind, .. } => {
+                self.calls += 1;
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.landing_pads += 1;
+                }
+            }
+            TerminatorKind::UnwindResume => self.resumes += 1,
+            TerminatorKind::InlineAsm { unwind, .. } => {
+                self.statements += 1;
+                if let UnwindAction::Cleanup(_) = unwind {
+                    self.landing_pads += 1;
+                }
+            }
+            TerminatorKind::Return => {}
+            _ => self.statements += 1,
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 06ae070c908..a70371172f7 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -169,8 +169,11 @@ impl<'tcx> Inliner<'tcx> {
         caller_body: &mut Body<'tcx>,
         callsite: &CallSite<'tcx>,
     ) -> Result<std::ops::Range<BasicBlock>, &'static str> {
+        self.check_mir_is_available(caller_body, &callsite.callee)?;
+
         let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
-        self.check_codegen_attributes(callsite, callee_attrs)?;
+        let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
+        self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
 
         let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
         let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
@@ -183,9 +186,8 @@ impl<'tcx> Inliner<'tcx> {
             }
         }
 
-        self.check_mir_is_available(caller_body, &callsite.callee)?;
         let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?;
-        self.check_mir_body(callsite, callee_body, callee_attrs)?;
+        self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;
 
         if !self.tcx.consider_optimizing(|| {
             format!("Inline {:?} into {:?}", callsite.callee, caller_body.source)
@@ -401,6 +403,7 @@ impl<'tcx> Inliner<'tcx> {
         &self,
         callsite: &CallSite<'tcx>,
         callee_attrs: &CodegenFnAttrs,
+        cross_crate_inlinable: bool,
     ) -> Result<(), &'static str> {
         if let InlineAttr::Never = callee_attrs.inline {
             return Err("never inline hint");
@@ -414,7 +417,7 @@ impl<'tcx> Inliner<'tcx> {
             .non_erasable_generics(self.tcx, callsite.callee.def_id())
             .next()
             .is_some();
-        if !is_generic && !callee_attrs.requests_inline() {
+        if !is_generic && !cross_crate_inlinable {
             return Err("not exported");
         }
 
@@ -456,10 +459,11 @@ impl<'tcx> Inliner<'tcx> {
         callsite: &CallSite<'tcx>,
         callee_body: &Body<'tcx>,
         callee_attrs: &CodegenFnAttrs,
+        cross_crate_inlinable: bool,
     ) -> Result<(), &'static str> {
         let tcx = self.tcx;
 
-        let mut threshold = if callee_attrs.requests_inline() {
+        let mut threshold = if cross_crate_inlinable {
             self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
         } else {
             self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 78f76f1e513..01136974730 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -62,6 +62,7 @@ mod const_prop;
 mod const_prop_lint;
 mod copy_prop;
 mod coverage;
+mod cross_crate_inline;
 mod ctfe_limit;
 mod dataflow_const_prop;
 mod dead_store_elimination;
@@ -123,6 +124,7 @@ pub fn provide(providers: &mut Providers) {
     coverage::query::provide(providers);
     ffi_unwind_calls::provide(providers);
     shim::provide(providers);
+    cross_crate_inline::provide(providers);
     *providers = Providers {
         mir_keys,
         mir_const,
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index e93368a84ba..650bb97c4d1 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -18,43 +18,10 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::CrateType;
 use rustc_target::spec::abi::Abi;
 
-// Returns true if the given item must be inlined because it may be
-// monomorphized or it was marked with `#[inline]`. This will only return
-// true for functions.
-fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool {
-    if attrs.requests_inline() {
-        return true;
-    }
-
-    match item.kind {
-        hir::ItemKind::Fn(ref sig, ..) if sig.header.is_const() => true,
-        hir::ItemKind::Impl { .. } | hir::ItemKind::Fn(..) => {
-            let generics = tcx.generics_of(item.owner_id);
-            generics.requires_monomorphization(tcx)
-        }
-        _ => false,
-    }
-}
-
-fn method_might_be_inlined(
-    tcx: TyCtxt<'_>,
-    impl_item: &hir::ImplItem<'_>,
-    impl_src: LocalDefId,
-) -> bool {
-    let codegen_fn_attrs = tcx.codegen_fn_attrs(impl_item.hir_id().owner.to_def_id());
-    let generics = tcx.generics_of(impl_item.owner_id);
-    if codegen_fn_attrs.requests_inline() || generics.requires_monomorphization(tcx) {
-        return true;
-    }
-    if let hir::ImplItemKind::Fn(method_sig, _) = &impl_item.kind {
-        if method_sig.header.is_const() {
-            return true;
-        }
-    }
-    match tcx.hir().find_by_def_id(impl_src) {
-        Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, codegen_fn_attrs),
-        Some(..) | None => span_bug!(impl_item.span, "impl did is not an item"),
-    }
+fn item_might_be_inlined(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+    tcx.generics_of(def_id).requires_monomorphization(tcx)
+        || tcx.cross_crate_inlinable(def_id)
+        || tcx.is_const_fn(def_id)
 }
 
 // Information needed while computing reachability.
@@ -150,9 +117,7 @@ impl<'tcx> ReachableContext<'tcx> {
 
         match self.tcx.hir().find_by_def_id(def_id) {
             Some(Node::Item(item)) => match item.kind {
-                hir::ItemKind::Fn(..) => {
-                    item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id))
-                }
+                hir::ItemKind::Fn(..) => item_might_be_inlined(self.tcx, def_id.into()),
                 _ => false,
             },
             Some(Node::TraitItem(trait_method)) => match trait_method.kind {
@@ -164,9 +129,7 @@ impl<'tcx> ReachableContext<'tcx> {
             Some(Node::ImplItem(impl_item)) => match impl_item.kind {
                 hir::ImplItemKind::Const(..) => true,
                 hir::ImplItemKind::Fn(..) => {
-                    let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
-                    let impl_did = self.tcx.hir().get_parent_item(hir_id);
-                    method_might_be_inlined(self.tcx, impl_item, impl_did.def_id)
+                    item_might_be_inlined(self.tcx, impl_item.hir_id().owner.to_def_id())
                 }
                 hir::ImplItemKind::Type(_) => false,
             },
@@ -226,11 +189,7 @@ impl<'tcx> ReachableContext<'tcx> {
             Node::Item(item) => {
                 match item.kind {
                     hir::ItemKind::Fn(.., body) => {
-                        if item_might_be_inlined(
-                            self.tcx,
-                            &item,
-                            self.tcx.codegen_fn_attrs(item.owner_id),
-                        ) {
+                        if item_might_be_inlined(self.tcx, item.owner_id.into()) {
                             self.visit_nested_body(body);
                         }
                     }
@@ -279,8 +238,7 @@ impl<'tcx> ReachableContext<'tcx> {
                     self.visit_nested_body(body);
                 }
                 hir::ImplItemKind::Fn(_, body) => {
-                    let impl_def_id = self.tcx.local_parent(search_item);
-                    if method_might_be_inlined(self.tcx, impl_item, impl_def_id) {
+                    if item_might_be_inlined(self.tcx, impl_item.hir_id().owner.to_def_id()) {
                         self.visit_nested_body(body)
                     }
                 }
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index f33139c5c4b..11fbd9f8f86 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1452,6 +1452,8 @@ options! {
         "combine CGUs into a single one"),
     crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
         "inject the given attribute in the crate"),
+    cross_crate_inline_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
+        "threshold to allow cross crate inlining of functions"),
     debug_info_for_profiling: bool = (false, parse_bool, [TRACKED],
         "emit discriminators and other data necessary for AutoFDO"),
     debug_macros: bool = (false, parse_bool, [TRACKED],
generated by cgit-pink 1.4.1-3-g733a5 (git 2.37.1) at 2026-09-15 18:33:48 +0000