Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make intrinsic fallback bodies cross-crate inlineable #121309

Merged
merged 1 commit into from
Feb 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions compiler/rustc_mir_transform/src/cross_crate_inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use rustc_middle::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::InliningThreshold;
use rustc_session::config::OptLevel;
use rustc_span::sym;

pub fn provide(providers: &mut Providers) {
providers.cross_crate_inlinable = cross_crate_inlinable;
Expand All @@ -34,6 +35,14 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
return true;
}

if tcx.has_attr(def_id, sym::rustc_intrinsic) {
// Intrinsic fallback bodies are always cross-crate inlineable.
// To ensure that the MIR inliner doesn't cluelessly try to inline fallback
// bodies even when the backend would implement something better, we stop
// the MIR inliner from ever inlining an intrinsic.
return true;
}

// Obey source annotations first; this is important because it means we can use
// #[inline(never)] to force code generation.
match codegen_fn_attrs.inline {
Expand Down
8 changes: 8 additions & 0 deletions compiler/rustc_mir_transform/src/inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use rustc_middle::ty::TypeVisitableExt;
use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
use rustc_session::config::OptLevel;
use rustc_span::source_map::Spanned;
use rustc_span::sym;
use rustc_target::abi::FieldIdx;
use rustc_target::spec::abi::Abi;

Expand Down Expand Up @@ -170,6 +171,13 @@ impl<'tcx> Inliner<'tcx> {
let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;

// Intrinsic fallback bodies are automatically made cross-crate inlineable,
// but at this stage we don't know whether codegen knows the intrinsic,
// so just conservatively don't inline it.
if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {
return Err("Callee is an intrinsic, do not inline fallback bodies");
}

let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty;
Expand Down
6 changes: 6 additions & 0 deletions library/core/src/intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,7 @@ extern "rust-intrinsic" {
#[rustc_nounwind]
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn assume(b: bool) {
if !b {
// SAFETY: the caller must guarantee the argument is never `false`
Expand All @@ -975,6 +976,7 @@ pub const unsafe fn assume(b: bool) {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[rustc_nounwind]
#[cfg_attr(bootstrap, inline)]
pub const fn likely(b: bool) -> bool {
b
}
Expand All @@ -994,6 +996,7 @@ pub const fn likely(b: bool) -> bool {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[rustc_nounwind]
#[cfg_attr(bootstrap, inline)]
pub const fn unlikely(b: bool) -> bool {
b
}
Expand Down Expand Up @@ -2556,6 +2559,7 @@ extern "rust-intrinsic" {
#[rustc_nounwind]
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
false
}
Expand Down Expand Up @@ -2592,6 +2596,7 @@ pub(crate) const fn debug_assertions() -> bool {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_nounwind]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
// const eval overrides this function, but runtime code should always just return null pointers.
crate::ptr::null_mut()
Expand All @@ -2611,6 +2616,7 @@ pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_nounwind]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {}

// Some functions are defined here because they accidentally got made
Expand Down
2 changes: 2 additions & 0 deletions src/doc/unstable-book/src/language-features/intrinsics.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ the intrinsic directly when you can.
Many intrinsics can be written in pure rust, albeit inefficiently or without supporting
some features that only exist on some backends. Backends can simply not implement those
intrinsics without causing any code miscompilations or failures to compile.
All intrinsic fallback bodies are automatically made cross-crate inlineable (like `#[inline]`)
by the codegen backend, but not the MIR inliner.

```rust
#![feature(rustc_attrs, effects)]
Expand Down
Loading