Skip to content

Commit

Permalink
Use insertvalue and extractvalue instead of memcpy in CastTarget
Browse files Browse the repository at this point in the history
  • Loading branch information
@DianQK
DianQK committed Aug 31, 2024
1 parent 33d3787 commit 02f6dd1
Show file tree
Hide file tree
Showing 10 changed files with 680 additions and 422 deletions.
34 changes: 2 additions & 32 deletions compiler/rustc_codegen_llvm/src/abi.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
use std::cmp;

use libc::c_uint;
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
use rustc_codegen_ssa::traits::*;
use rustc_codegen_ssa::MemFlags;
use rustc_middle::ty::layout::LayoutOf;
pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
use rustc_middle::ty::Ty;
Expand Down Expand Up @@ -215,35 +212,8 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
}
PassMode::Cast { cast, pad_i32: _ } => {
// The ABI mandates that the value is passed as a different struct representation.
// Spill and reload it from the stack to convert from the ABI representation to
// the Rust representation.
let scratch_size = cast.size(bx);
let scratch_align = cast.align(bx);
// Note that the ABI type may be either larger or smaller than the Rust type,
// due to the presence or absence of trailing padding. For example:
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
// when passed by value, making it smaller.
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
// when passed by value, making it larger.
let copy_bytes =
cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes());
// Allocate some scratch space...
let llscratch = bx.alloca(scratch_size, scratch_align);
bx.lifetime_start(llscratch, scratch_size);
// ...store the value...
bx.store(val, llscratch, scratch_align);
// ... and then memcpy it to the intended destination.
bx.memcpy(
dst.val.llval,
self.layout.align.abi,
llscratch,
scratch_align,
bx.const_usize(copy_bytes),
MemFlags::empty(),
);
bx.lifetime_end(llscratch, scratch_size);
PassMode::Cast { cast, .. } => {
cast.cast_other_abi_to_rust(bx, val, dst.val.llval, self.layout);
}
_ => {
OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
Expand Down
122 changes: 35 additions & 87 deletions compiler/rustc_codegen_ssa/src/mir/block.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ use super::{CachedLlbb, FunctionCx, LocalRef};
use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
use crate::common::{self, IntPredicate};
use crate::errors::CompilerBuiltinsCannotCall;
use crate::meth;
use crate::traits::*;
use crate::{meth, MemFlags};

// Indicates if we are in the middle of merging a BB's successor into it. This
// can happen when BB jumps directly to its successor and the successor has no
Expand Down Expand Up @@ -462,7 +462,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}

PassMode::Cast { cast: cast_ty, pad_i32: _ } => {
PassMode::Cast { cast, pad_i32: _ } => {
let op = match self.locals[mir::RETURN_PLACE] {
LocalRef::Operand(op) => op,
LocalRef::PendingOperand => bug!("use of return before def"),
Expand All @@ -471,23 +471,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
};
let llslot = match op.val {
Immediate(_) | Pair(..) => {
let scratch = PlaceRef::alloca(bx, self.fn_abi.ret.layout);
op.val.store(bx, scratch);
scratch.val.llval
}
Ref(place_val) => {
assert_eq!(
place_val.align, op.layout.align.abi,
"return place is unaligned!"
);
place_val.llval
}
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
};
let ty = bx.cast_backend_type(cast_ty);
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
cast.cast_rust_abi_to_other(bx, op)
}
};
bx.ret(llval);
Expand Down Expand Up @@ -1460,10 +1444,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
_ => {}
}

// Force by-ref if we have to load through a cast pointer.
let (mut llval, align, by_ref) = match op.val {
Immediate(_) | Pair(..) => match arg.mode {
PassMode::Indirect { attrs, .. } => {
let llval = match arg.mode {
PassMode::Indirect { attrs, on_stack, .. } => match op.val {
Immediate(_) | Pair(..) => {
// Indirect argument may have higher alignment requirements than the type's alignment.
// This can happen, e.g. when passing types with <4 byte alignment on the stack on x86.
let required_align = match attrs.pointee_align {
Expand All @@ -1472,17 +1455,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
};
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
op.val.store(bx, scratch.with_type(arg.layout));
(scratch.llval, scratch.align, true)
}
PassMode::Cast { .. } => {
let scratch = PlaceRef::alloca(bx, arg.layout);
op.val.store(bx, scratch);
(scratch.val.llval, scratch.val.align, true)
scratch.llval
}
_ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false),
},
Ref(op_place_val) => match arg.mode {
PassMode::Indirect { attrs, .. } => {
Ref(op_place_val) => {
let required_align = match attrs.pointee_align {
Some(pointee_align) => cmp::max(pointee_align, arg.layout.align.abi),
None => arg.layout.align.abi,
Expand All @@ -1493,15 +1468,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// to a higher-aligned alloca.
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
bx.typed_place_copy(scratch, op_place_val, op.layout);
(scratch.llval, scratch.align, true)
scratch.llval
} else {
(op_place_val.llval, op_place_val.align, true)
op_place_val.llval
}
}
_ => (op_place_val.llval, op_place_val.align, true),
},
ZeroSized => match arg.mode {
PassMode::Indirect { on_stack, .. } => {
ZeroSized => {
if on_stack {
// It doesn't seem like any target can have `byval` ZSTs, so this assert
// is here to replace a would-be untested codepath.
Expand All @@ -1511,59 +1483,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// a pointer for `repr(C)` structs even when empty, so get
// one from an `alloca` (which can be left uninitialized).
let scratch = PlaceRef::alloca(bx, arg.layout);
(scratch.val.llval, scratch.val.align, true)
scratch.val.llval
}
_ => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"),
},
};

if by_ref && !arg.is_indirect() {
// Have to load the argument, maybe while casting it.
if let PassMode::Cast { cast, pad_i32: _ } = &arg.mode {
// The ABI mandates that the value is passed as a different struct representation.
// Spill and reload it from the stack to convert from the Rust representation to
// the ABI representation.
let scratch_size = cast.size(bx);
let scratch_align = cast.align(bx);
// Note that the ABI type may be either larger or smaller than the Rust type,
// due to the presence or absence of trailing padding. For example:
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
// when passed by value, making it smaller.
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
// when passed by value, making it larger.
let copy_bytes = cmp::min(cast.unaligned_size(bx).bytes(), arg.layout.size.bytes());
// Allocate some scratch space...
let llscratch = bx.alloca(scratch_size, scratch_align);
bx.lifetime_start(llscratch, scratch_size);
// ...memcpy the value...
bx.memcpy(
llscratch,
scratch_align,
llval,
align,
bx.const_usize(copy_bytes),
MemFlags::empty(),
);
// ...and then load it with the ABI type.
let cast_ty = bx.cast_backend_type(cast);
llval = bx.load(cast_ty, llscratch, scratch_align);
bx.lifetime_end(llscratch, scratch_size);
} else {
// We can't use `PlaceRef::load` here because the argument
// may have a type we don't treat as immediate, but the ABI
// used for this call is passing it by-value. In that case,
// the load would just produce `OperandValue::Ref` instead
// of the `OperandValue::Immediate` we need for the call.
llval = bx.load(bx.backend_type(arg.layout), llval, align);
if let abi::Abi::Scalar(scalar) = arg.layout.abi {
if scalar.is_bool() {
PassMode::Cast { ref cast, .. } => cast.cast_rust_abi_to_other(bx, op),
_ => match op.val {
Immediate(_) | Pair(..) => op.immediate_or_packed_pair(bx),
Ref(op_place_val) => {
// We can't use `PlaceRef::load` here because the argument
// may have a type we don't treat as immediate, but the ABI
// used for this call is passing it by-value. In that case,
// the load would just produce `OperandValue::Ref` instead
// of the `OperandValue::Immediate` we need for the call.
let mut llval = bx.load(
bx.backend_type(arg.layout),
op_place_val.llval,
op_place_val.align,
);
if let abi::Abi::Scalar(scalar) = arg.layout.abi
&& scalar.is_bool()
{
bx.range_metadata(llval, WrappingRange { start: 0, end: 1 });
}
// We store bools as `i8` so we need to truncate to `i1`.
llval = bx.to_immediate(llval, arg.layout);
llval
}
// We store bools as `i8` so we need to truncate to `i1`.
llval = bx.to_immediate(llval, arg.layout);
}
}
ZeroSized => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"),
},
};

llargs.push(llval);
}
Expand Down
18 changes: 4 additions & 14 deletions compiler/rustc_codegen_ssa/src/mir/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,20 +229,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let layout = start_bx.layout_of(fx.monomorphize(decl.ty));
assert!(!layout.ty.has_erasable_regions());

if local == mir::RETURN_PLACE {
match fx.fn_abi.ret.mode {
PassMode::Indirect { .. } => {
debug!("alloc: {:?} (return place) -> place", local);
let llretptr = start_bx.get_param(0);
return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
}
PassMode::Cast { ref cast, .. } => {
debug!("alloc: {:?} (return place) -> place", local);
let size = cast.size(&start_bx);
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
}
_ => {}
};
if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() {
debug!("alloc: {:?} (return place) -> place", local);
let llretptr = start_bx.get_param(0);
return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
}

if memory_locals.contains(local) {
Expand Down
Loading

0 comments on commit 02f6dd1

Please sign in to comment.