Auto merge of rust-lang#130679 - saethlin:inline-usually, r=<try>

Add inline(usually) r? `@ghost` I'm looking into what kind of things could recover the perf improvement detected in rust-lang#121417 (comment)
rust-lang-ci · Sep 22, 2024 · 31d3b5c · 31d3b5c
2 parents 0af7f0f + e4ff49d
commit 31d3b5c
Show file tree

Hide file tree

Showing 47 changed files with 586 additions and 347 deletions.
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
@@ -46,6 +46,7 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    Usually,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs
@@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>(
                 None
             }
         }
+        InlineAttr::Usually => Some(FnAttribute::Inline),
         InlineAttr::None => None,
     }
 }

diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
             }
         }
         InlineAttr::None => None,
+        InlineAttr::Usually => {
+            Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
+        }
     }
 }
 

diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                         .emit();
                     InlineAttr::None
                 } else if list_contains_name(items, sym::always) {
-                    InlineAttr::Always
+                    InlineAttr::Usually
                 } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
+                } else if list_contains_name(items, sym::usually) {
+                    InlineAttr::Usually
                 } else {
                     struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                         .with_help("valid inline arguments are `always` and `never`")

diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
         _ => {}
     }
 

diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
         changed: false,
         caller_is_inline_forwarder: matches!(
             codegen_fn_attrs.inline,
-            InlineAttr::Hint | InlineAttr::Always
+            InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
         ) && body_is_forwarder(body),
     };
     let blocks = START_BLOCK..body.basic_blocks.next_index();

diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
@@ -2101,6 +2101,7 @@ symbols! {
         usize_legacy_fn_max_value,
         usize_legacy_fn_min_value,
         usize_legacy_mod,
+        usually,
         va_arg,
         va_copy,
         va_end,

diff --git a/library/core/src/alloc/layout.rs b/library/core/src/alloc/layout.rs
@@ -84,7 +84,8 @@ impl Layout {
         true
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     const fn max_size_for_align(align: Alignment) -> usize {
         // (power-of-two implies align != 0.)
 

diff --git a/library/core/src/alloc/mod.rs b/library/core/src/alloc/mod.rs
@@ -360,7 +360,8 @@ pub unsafe trait Allocator {
     /// Creates a "by reference" adapter for this instance of `Allocator`.
     ///
     /// The returned adapter also implements `Allocator` and will simply borrow this.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn by_ref(&self) -> &Self
     where
         Self: Sized,

diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
@@ -813,12 +813,14 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
 type BorrowFlag = isize;
 const UNUSED: BorrowFlag = 0;
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]
+#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_writing(x: BorrowFlag) -> bool {
     x < UNUSED
 }
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]
+#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_reading(x: BorrowFlag) -> bool {
     x > UNUSED
 }
@@ -2079,7 +2081,8 @@ impl<T> UnsafeCell<T> {
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     pub const fn new(value: T) -> UnsafeCell<T> {
         UnsafeCell { value }
     }
@@ -2095,7 +2098,8 @@ impl<T> UnsafeCell<T> {
     ///
     /// let five = uc.into_inner();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     // When this is const stabilized, please remove `primitive_into_inner` below.
     #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")]
@@ -2119,7 +2123,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     /// *uc.get_mut() -= 1;
     /// assert_eq!(*uc.get_mut(), 41);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[unstable(feature = "unsafe_cell_from_mut", issue = "111645")]
     pub const fn from_mut(value: &mut T) -> &mut UnsafeCell<T> {
         // SAFETY: `UnsafeCell<T>` has the same memory layout as `T` due to #[repr(transparent)].
@@ -2142,7 +2147,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// let five = uc.get();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
     #[rustc_never_returns_null_ptr]
@@ -2168,7 +2174,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(*c.get_mut(), 6);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")]
     #[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")]
     pub const fn get_mut(&mut self) -> &mut T {
@@ -2203,7 +2210,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(uc.into_inner(), 5);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_diagnostic_item = "unsafe_cell_raw_get"]

diff --git a/library/core/src/clone.rs b/library/core/src/clone.rs
@@ -321,7 +321,7 @@ mod impls {
             $(
                 #[stable(feature = "rust1", since = "1.0.0")]
                 impl Clone for $t {
-                    #[inline(always)]
+                    #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                     fn clone(&self) -> Self {
                         *self
                     }
@@ -347,15 +347,17 @@ mod impls {
 
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for *const T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]
+        #[cfg_attr(not(bootstrap), inline(usually))]
         fn clone(&self) -> Self {
             *self
         }
     }
 
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for *mut T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]
+        #[cfg_attr(not(bootstrap), inline(usually))]
         fn clone(&self) -> Self {
             *self
         }
@@ -364,7 +366,8 @@ mod impls {
     /// Shared references can be cloned, but mutable references *cannot*!
     #[stable(feature = "rust1", since = "1.0.0")]
     impl<T: ?Sized> Clone for &T {
-        #[inline(always)]
+        #[cfg_attr(bootstrap, inline(always))]
+        #[cfg_attr(not(bootstrap), inline(usually))]
         #[rustc_diagnostic_item = "noop_method_clone"]
         fn clone(&self) -> Self {
             *self

diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs
@@ -1520,13 +1520,13 @@ mod impls {
                         (true, true) => Some(Equal),
                     }
                 }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn le(&self, other: &$t) -> bool { (*self) <= (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn ge(&self, other: &$t) -> bool { (*self) >= (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
             }
         )*)
@@ -1558,13 +1558,13 @@ mod impls {
                 fn partial_cmp(&self, other: &$t) -> Option<Ordering> {
                     Some(crate::intrinsics::three_way_compare(*self, *other))
                 }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn le(&self, other: &$t) -> bool { (*self) <= (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn ge(&self, other: &$t) -> bool { (*self) >= (*other) }
-                #[inline(always)]
+                #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
                 fn gt(&self, other: &$t) -> bool { (*self) > (*other) }
             }
 

diff --git a/library/core/src/convert/mod.rs b/library/core/src/convert/mod.rs
@@ -99,7 +99,8 @@ pub use num::FloatToInt;
 /// ```
 #[stable(feature = "convert_id", since = "1.33.0")]
 #[rustc_const_stable(feature = "const_identity", since = "1.33.0")]
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]
+#[cfg_attr(not(bootstrap), inline(usually))]
 #[rustc_diagnostic_item = "convert_identity"]
 pub const fn identity<T>(x: T) -> T {
     x
@@ -764,7 +765,8 @@ where
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<T> From<T> for T {
     /// Returns the argument unchanged.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn from(t: T) -> T {
         t
     }
@@ -820,31 +822,35 @@ where
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<T> AsRef<[T]> for [T] {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn as_ref(&self) -> &[T] {
         self
     }
 }
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl<T> AsMut<[T]> for [T] {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn as_mut(&mut self) -> &mut [T] {
         self
     }
 }
 
 #[stable(feature = "rust1", since = "1.0.0")]
 impl AsRef<str> for str {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn as_ref(&self) -> &str {
         self
     }
 }
 
 #[stable(feature = "as_mut_str_for_str", since = "1.51.0")]
 impl AsMut<str> for str {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn as_mut(&mut self) -> &mut str {
         self
     }

diff --git a/library/core/src/convert/num.rs b/library/core/src/convert/num.rs
@@ -73,7 +73,7 @@ macro_rules! impl_from {
             // Rustdocs on the impl block show a "[+] show undocumented items" toggle.
             // Rustdocs on functions do not.
             #[doc = $doc]
-            #[inline(always)]
+            #[cfg_attr(bootstrap, inline(always))]#[cfg_attr(not(bootstrap), inline(usually))]
             fn from(small: $Small) -> Self {
                 small as Self
             }

diff --git a/library/core/src/default.rs b/library/core/src/default.rs
@@ -151,7 +151,8 @@ macro_rules! default_impl {
     ($t:ty, $v:expr, $doc:tt) => {
         #[stable(feature = "rust1", since = "1.0.0")]
         impl Default for $t {
-            #[inline(always)]
+            #[cfg_attr(bootstrap, inline(always))]
+            #[cfg_attr(not(bootstrap), inline(usually))]
             #[doc = $doc]
             fn default() -> $t {
                 $v