Skip to content

Commit f1cbf12

Browse files
borsbors
committed
Auto merge of #116505 - saethlin:infer-inline, r=<try>
Automatically enable cross-crate inlining for small functions This is a work-in-progress. For example I have not thought at all about the cost model and I am sure that the threshold is too high. But I'm curious to know how this looks in perf. It certainly has some unique effects on codegen.
2 parents 4ea5190 + 64f45aa commit f1cbf12

21 files changed

+379
-158
lines changed

‎compiler/rustc_metadata/src/rmeta/decoder.rs

+9
Original file line numberDiff line numberDiff line change
@@ -1273,6 +1273,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
12731273
self.root.tables.optimized_mir.get(self, id).is_some()
12741274
}
12751275

1276+
fn cross_crate_inlinable(self, tcx: TyCtxt<'tcx>, id: DefIndex) -> bool {
1277+
self.root
1278+
.tables
1279+
.cross_crate_inlinable
1280+
.get(self, id)
1281+
.map(|v| v.decode((self, tcx.sess)))
1282+
.unwrap_or(false)
1283+
}
1284+
12761285
fn get_fn_has_self_parameter(self, id: DefIndex, sess: &'a Session) -> bool {
12771286
self.root
12781287
.tables

‎compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

+1
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ provide! { tcx, def_id, other, cdata,
287287
item_attrs => { tcx.arena.alloc_from_iter(cdata.get_item_attrs(def_id.index, tcx.sess)) }
288288
is_mir_available => { cdata.is_item_mir_available(def_id.index) }
289289
is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) }
290+
cross_crate_inlinable => { cdata.cross_crate_inlinable(tcx, def_id.index) }
290291

291292
dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) }
292293
is_private_dep => {

‎compiler/rustc_metadata/src/rmeta/encoder.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1046,7 +1046,7 @@ fn should_encode_mir(
10461046
|| (tcx.sess.opts.output_types.should_codegen()
10471047
&& reachable_set.contains(&def_id)
10481048
&& (generics.requires_monomorphization(tcx)
1049-
|| tcx.codegen_fn_attrs(def_id).requests_inline()));
1049+
|| tcx.cross_crate_inlinable(def_id)));
10501050
// The function has a `const` modifier or is in a `#[const_trait]`.
10511051
let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id())
10521052
|| tcx.is_const_default_method(def_id.to_def_id());
@@ -1612,6 +1612,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
16121612
debug!("EntryBuilder::encode_mir({:?})", def_id);
16131613
if encode_opt {
16141614
record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id));
1615+
record!(self.tables.cross_crate_inlinable[def_id.to_def_id()] <- self.tcx.cross_crate_inlinable(def_id));
16151616
record!(self.tables.closure_saved_names_of_captured_variables[def_id.to_def_id()]
16161617
<- tcx.closure_saved_names_of_captured_variables(def_id));
16171618

‎compiler/rustc_metadata/src/rmeta/mod.rs

+1
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ define_tables! {
427427
object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
428428
optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
429429
mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
430+
cross_crate_inlinable: Table<DefIndex, LazyValue<bool>>,
430431
closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
431432
mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
432433
promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,

‎compiler/rustc_metadata/src/rmeta/table.rs

+23
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,29 @@ impl FixedSizeEncoding for bool {
299299
}
300300
}
301301

302+
impl FixedSizeEncoding for Option<bool> {
303+
type ByteArray = [u8; 1];
304+
305+
#[inline]
306+
fn from_bytes(b: &[u8; 1]) -> Self {
307+
match b[0] {
308+
0 => Some(false),
309+
1 => Some(true),
310+
_ => None,
311+
}
312+
}
313+
314+
#[inline]
315+
fn write_to_bytes(self, b: &mut [u8; 1]) {
316+
debug_assert!(!self.is_default());
317+
b[0] = match self {
318+
Some(false) => 0,
319+
Some(true) => 1,
320+
None => 2,
321+
};
322+
}
323+
}
324+
302325
impl FixedSizeEncoding for UnusedGenericParams {
303326
type ByteArray = [u8; 4];
304327

‎compiler/rustc_middle/src/query/mod.rs

+5
Original file line numberDiff line numberDiff line change
@@ -2202,6 +2202,11 @@ rustc_queries! {
22022202
query generics_require_sized_self(def_id: DefId) -> bool {
22032203
desc { "check whether the item has a `where Self: Sized` bound" }
22042204
}
2205+
2206+
query cross_crate_inlinable(def_id: DefId) -> bool {
2207+
desc { "whether the item should be made inlinable across crates" }
2208+
separate_provide_extern
2209+
}
22052210
}
22062211

22072212
rustc_query_append! { define_callbacks! }

‎compiler/rustc_middle/src/ty/instance.rs

+4-5
Original file line numberDiff line numberDiff line change
@@ -245,16 +245,15 @@ impl<'tcx> InstanceDef<'tcx> {
245245
// drops of `Option::None` before LTO. We also respect the intent of
246246
// `#[inline]` on `Drop::drop` implementations.
247247
return ty.ty_adt_def().map_or(true, |adt_def| {
248-
adt_def.destructor(tcx).map_or_else(
249-
|| adt_def.is_enum(),
250-
|dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
251-
)
248+
adt_def
249+
.destructor(tcx)
250+
.map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
252251
});
253252
}
254253
if let ty::InstanceDef::ThreadLocalShim(..) = *self {
255254
return false;
256255
}
257-
tcx.codegen_fn_attrs(self.def_id()).requests_inline()
256+
tcx.cross_crate_inlinable(self.def_id())
258257
}
259258

260259
pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {

‎compiler/rustc_mir_transform/src/cross_crate_inline.rs

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
use rustc_attr::InlineAttr;
2+
use rustc_hir::def_id::LocalDefId;
3+
use rustc_middle::query::Providers;
4+
use rustc_middle::ty::TyCtxt;
5+
use rustc_session::config::OptLevel;
6+
7+
pub fn provide(providers: &mut Providers) {
8+
providers.cross_crate_inlinable = cross_crate_inlinable;
9+
}
10+
11+
fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
12+
match tcx.codegen_fn_attrs(def_id).inline {
13+
InlineAttr::Never => return false,
14+
InlineAttr::Hint | InlineAttr::Always => return true,
15+
_ => {}
16+
}
17+
18+
if matches!(tcx.sess.opts.optimize, OptLevel::No | OptLevel::Default) {
19+
return false;
20+
}
21+
22+
match tcx.hir().body_const_context(def_id) {
23+
Some(rustc_hir::ConstContext::ConstFn) | None => {}
24+
_ => return false,
25+
}
26+
27+
if tcx.lang_items().iter().any(|(_, lang_def_id)| lang_def_id == def_id.into()) {
28+
return false;
29+
}
30+
31+
let mir = tcx.optimized_mir(def_id);
32+
let mut checker = CostChecker { tcx, cost: 0, callee_body: mir };
33+
checker.visit_body(mir);
34+
checker.cost < 200
35+
}
36+
37+
use rustc_middle::mir::visit::Visitor;
38+
use rustc_middle::mir::*;
39+
40+
const INSTR_COST: usize = 5;
41+
const CALL_PENALTY: usize = 25;
42+
const LANDINGPAD_PENALTY: usize = 50;
43+
const RESUME_PENALTY: usize = 45;
44+
45+
struct CostChecker<'b, 'tcx> {
46+
tcx: TyCtxt<'tcx>,
47+
cost: usize,
48+
callee_body: &'b Body<'tcx>,
49+
}
50+
51+
impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
52+
fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
53+
// Don't count StorageLive/StorageDead in the inlining cost.
54+
match statement.kind {
55+
StatementKind::StorageLive(_)
56+
| StatementKind::StorageDead(_)
57+
| StatementKind::Deinit(_)
58+
| StatementKind::Nop => {}
59+
_ => self.cost += INSTR_COST,
60+
}
61+
}
62+
63+
fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
64+
let tcx = self.tcx;
65+
match terminator.kind {
66+
TerminatorKind::Drop { ref place, unwind, .. } => {
67+
let ty = place.ty(self.callee_body, tcx).ty;
68+
if !ty.is_trivially_pure_clone_copy() {
69+
self.cost += CALL_PENALTY;
70+
if let UnwindAction::Cleanup(_) = unwind {
71+
self.cost += LANDINGPAD_PENALTY;
72+
}
73+
}
74+
}
75+
TerminatorKind::Call { unwind, .. } => {
76+
self.cost += CALL_PENALTY;
77+
if let UnwindAction::Cleanup(_) = unwind {
78+
self.cost += LANDINGPAD_PENALTY;
79+
}
80+
}
81+
TerminatorKind::Assert { unwind, .. } => {
82+
self.cost += CALL_PENALTY;
83+
if let UnwindAction::Cleanup(_) = unwind {
84+
self.cost += LANDINGPAD_PENALTY;
85+
}
86+
}
87+
TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY,
88+
TerminatorKind::InlineAsm { unwind, .. } => {
89+
self.cost += INSTR_COST;
90+
if let UnwindAction::Cleanup(_) = unwind {
91+
self.cost += LANDINGPAD_PENALTY;
92+
}
93+
}
94+
_ => self.cost += INSTR_COST,
95+
}
96+
}
97+
}

‎compiler/rustc_mir_transform/src/inline.rs

+12-4
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,13 @@ impl<'tcx> Inliner<'tcx> {
170170
callsite: &CallSite<'tcx>,
171171
) -> Result<std::ops::Range<BasicBlock>, &'static str> {
172172
let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
173-
self.check_codegen_attributes(callsite, callee_attrs)?;
173+
let cross_crate_inlinable = if callsite.callee.def_id().is_local() {
174+
// Avoid a query cycle, cross_crate_inlinable is based on optimized_mir
175+
callee_attrs.requests_inline()
176+
} else {
177+
self.tcx.cross_crate_inlinable(callsite.callee.def_id())
178+
};
179+
self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
174180

175181
let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
176182
let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
@@ -185,7 +191,7 @@ impl<'tcx> Inliner<'tcx> {
185191

186192
self.check_mir_is_available(caller_body, &callsite.callee)?;
187193
let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?;
188-
self.check_mir_body(callsite, callee_body, callee_attrs)?;
194+
self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;
189195

190196
if !self.tcx.consider_optimizing(|| {
191197
format!("Inline {:?} into {:?}", callsite.callee, caller_body.source)
@@ -401,6 +407,7 @@ impl<'tcx> Inliner<'tcx> {
401407
&self,
402408
callsite: &CallSite<'tcx>,
403409
callee_attrs: &CodegenFnAttrs,
410+
cross_crate_inlinable: bool,
404411
) -> Result<(), &'static str> {
405412
if let InlineAttr::Never = callee_attrs.inline {
406413
return Err("never inline hint");
@@ -414,7 +421,7 @@ impl<'tcx> Inliner<'tcx> {
414421
.non_erasable_generics(self.tcx, callsite.callee.def_id())
415422
.next()
416423
.is_some();
417-
if !is_generic && !callee_attrs.requests_inline() {
424+
if !is_generic && !cross_crate_inlinable {
418425
return Err("not exported");
419426
}
420427

@@ -456,10 +463,11 @@ impl<'tcx> Inliner<'tcx> {
456463
callsite: &CallSite<'tcx>,
457464
callee_body: &Body<'tcx>,
458465
callee_attrs: &CodegenFnAttrs,
466+
cross_crate_inlinable: bool,
459467
) -> Result<(), &'static str> {
460468
let tcx = self.tcx;
461469

462-
let mut threshold = if callee_attrs.requests_inline() {
470+
let mut threshold = if cross_crate_inlinable {
463471
self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
464472
} else {
465473
self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)

‎compiler/rustc_mir_transform/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ mod const_prop;
6262
mod const_prop_lint;
6363
mod copy_prop;
6464
mod coverage;
65+
mod cross_crate_inline;
6566
mod ctfe_limit;
6667
mod dataflow_const_prop;
6768
mod dead_store_elimination;
@@ -123,6 +124,7 @@ pub fn provide(providers: &mut Providers) {
123124
coverage::query::provide(providers);
124125
ffi_unwind_calls::provide(providers);
125126
shim::provide(providers);
127+
cross_crate_inline::provide(providers);
126128
*providers = Providers {
127129
mir_keys,
128130
mir_const,

‎compiler/rustc_passes/src/reachable.rs

+7-7
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ use rustc_target::spec::abi::Abi;
2121
// Returns true if the given item must be inlined because it may be
2222
// monomorphized or it was marked with `#[inline]`. This will only return
2323
// true for functions.
24-
fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool {
25-
if attrs.requests_inline() {
24+
fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, inlinable: bool) -> bool {
25+
if inlinable {
2626
return true;
2727
}
2828

@@ -41,9 +41,9 @@ fn method_might_be_inlined(
4141
impl_item: &hir::ImplItem<'_>,
4242
impl_src: LocalDefId,
4343
) -> bool {
44-
let codegen_fn_attrs = tcx.codegen_fn_attrs(impl_item.hir_id().owner.to_def_id());
44+
let inlinable = tcx.cross_crate_inlinable(impl_item.hir_id().owner.to_def_id());
4545
let generics = tcx.generics_of(impl_item.owner_id);
46-
if codegen_fn_attrs.requests_inline() || generics.requires_monomorphization(tcx) {
46+
if inlinable || generics.requires_monomorphization(tcx) {
4747
return true;
4848
}
4949
if let hir::ImplItemKind::Fn(method_sig, _) = &impl_item.kind {
@@ -52,7 +52,7 @@ fn method_might_be_inlined(
5252
}
5353
}
5454
match tcx.hir().find_by_def_id(impl_src) {
55-
Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, codegen_fn_attrs),
55+
Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, inlinable),
5656
Some(..) | None => span_bug!(impl_item.span, "impl did is not an item"),
5757
}
5858
}
@@ -149,7 +149,7 @@ impl<'tcx> ReachableContext<'tcx> {
149149
match self.tcx.hir().find_by_def_id(def_id) {
150150
Some(Node::Item(item)) => match item.kind {
151151
hir::ItemKind::Fn(..) => {
152-
item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id))
152+
item_might_be_inlined(self.tcx, &item, self.tcx.cross_crate_inlinable(def_id))
153153
}
154154
_ => false,
155155
},
@@ -227,7 +227,7 @@ impl<'tcx> ReachableContext<'tcx> {
227227
if item_might_be_inlined(
228228
self.tcx,
229229
&item,
230-
self.tcx.codegen_fn_attrs(item.owner_id),
230+
self.tcx.cross_crate_inlinable(item.owner_id),
231231
) {
232232
self.visit_nested_body(body);
233233
}

‎tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
}
6767
+ }
6868
+
69-
+ alloc15 (size: 8, align: 4) {
69+
+ alloc14 (size: 8, align: 4) {
7070
+ 02 00 00 00 05 20 00 00 │ ..... ..
7171
}
7272

‎tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
}
6767
+ }
6868
+
69-
+ alloc15 (size: 16, align: 8) {
69+
+ alloc14 (size: 16, align: 8) {
7070
+ 02 00 00 00 00 00 00 00 05 20 00 00 00 00 00 00 │ ......... ......
7171
}
7272

‎tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
}
6767
+ }
6868
+
69-
+ alloc14 (size: 8, align: 4) {
69+
+ alloc15 (size: 8, align: 4) {
7070
+ 05 20 00 00 01 00 00 00 │ . ......
7171
}
7272

‎tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
}
6767
+ }
6868
+
69-
+ alloc14 (size: 16, align: 8) {
69+
+ alloc15 (size: 16, align: 8) {
7070
+ 05 20 00 00 00 00 00 00 01 00 00 00 00 00 00 00 │ . ..............
7171
}
7272

0 commit comments

Comments
 (0)