Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use insertvalue and extractvalue instead of memcpy in CastTarget #128969

Closed
wants to merge 2 commits into from
+682 −424
Closed

Use insertvalue and extractvalue instead of memcpy in CastTarget #128969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33d3787
Implement `reg_backend_type` for gcc
DianQK Aug 12, 2024
02f6dd1
Use `insertvalue` and `extractvalue` instead of `memcpy` in CastTarget
DianQK Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions compiler/rustc_codegen_gcc/src/type_of.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,8 +364,8 @@ impl<'gcc, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
fn_abi.ptr_to_gcc_type(self)
}

fn reg_backend_type(&self, _ty: &Reg) -> Type<'gcc> {
unimplemented!();
fn reg_backend_type(&self, ty: &Reg) -> Type<'gcc> {
ty.gcc_type(self)
}

fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
Expand Down
34 changes: 2 additions & 32 deletions compiler/rustc_codegen_llvm/src/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
use std::cmp;

use libc::c_uint;
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
use rustc_codegen_ssa::traits::*;
use rustc_codegen_ssa::MemFlags;
use rustc_middle::ty::layout::LayoutOf;
pub(crate) use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
use rustc_middle::ty::Ty;
Expand Down Expand Up @@ -215,35 +212,8 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
}
PassMode::Cast { cast, pad_i32: _ } => {
// The ABI mandates that the value is passed as a different struct representation.
// Spill and reload it from the stack to convert from the ABI representation to
// the Rust representation.
let scratch_size = cast.size(bx);
let scratch_align = cast.align(bx);
// Note that the ABI type may be either larger or smaller than the Rust type,
// due to the presence or absence of trailing padding. For example:
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
// when passed by value, making it smaller.
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
// when passed by value, making it larger.
let copy_bytes =
cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes());
// Allocate some scratch space...
let llscratch = bx.alloca(scratch_size, scratch_align);
bx.lifetime_start(llscratch, scratch_size);
// ...store the value...
bx.store(val, llscratch, scratch_align);
// ... and then memcpy it to the intended destination.
bx.memcpy(
dst.val.llval,
self.layout.align.abi,
llscratch,
scratch_align,
bx.const_usize(copy_bytes),
MemFlags::empty(),
);
bx.lifetime_end(llscratch, scratch_size);
PassMode::Cast { cast, .. } => {
cast.cast_other_abi_to_rust(bx, val, dst.val.llval, self.layout);
}
_ => {
OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
Expand Down
122 changes: 35 additions & 87 deletions compiler/rustc_codegen_ssa/src/mir/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ use super::{CachedLlbb, FunctionCx, LocalRef};
use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
use crate::common::{self, IntPredicate};
use crate::errors::CompilerBuiltinsCannotCall;
use crate::meth;
use crate::traits::*;
use crate::{meth, MemFlags};

// Indicates if we are in the middle of merging a BB's successor into it. This
// can happen when BB jumps directly to its successor and the successor has no
Expand Down Expand Up @@ -462,7 +462,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}

PassMode::Cast { cast: cast_ty, pad_i32: _ } => {
PassMode::Cast { cast, pad_i32: _ } => {
let op = match self.locals[mir::RETURN_PLACE] {
LocalRef::Operand(op) => op,
LocalRef::PendingOperand => bug!("use of return before def"),
Expand All @@ -471,23 +471,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
};
let llslot = match op.val {
Immediate(_) | Pair(..) => {
let scratch = PlaceRef::alloca(bx, self.fn_abi.ret.layout);
op.val.store(bx, scratch);
scratch.val.llval
}
Ref(place_val) => {
assert_eq!(
place_val.align, op.layout.align.abi,
"return place is unaligned!"
);
place_val.llval
}
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
};
let ty = bx.cast_backend_type(cast_ty);
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
cast.cast_rust_abi_to_other(bx, op)
}
};
bx.ret(llval);
Expand Down Expand Up @@ -1460,10 +1444,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
_ => {}
}

// Force by-ref if we have to load through a cast pointer.
let (mut llval, align, by_ref) = match op.val {
Immediate(_) | Pair(..) => match arg.mode {
PassMode::Indirect { attrs, .. } => {
let llval = match arg.mode {
PassMode::Indirect { attrs, on_stack, .. } => match op.val {
Immediate(_) | Pair(..) => {
// Indirect argument may have higher alignment requirements than the type's alignment.
// This can happen, e.g. when passing types with <4 byte alignment on the stack on x86.
let required_align = match attrs.pointee_align {
Expand All @@ -1472,17 +1455,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
};
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
op.val.store(bx, scratch.with_type(arg.layout));
(scratch.llval, scratch.align, true)
}
PassMode::Cast { .. } => {
let scratch = PlaceRef::alloca(bx, arg.layout);
op.val.store(bx, scratch);
(scratch.val.llval, scratch.val.align, true)
scratch.llval
}
_ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false),
},
Ref(op_place_val) => match arg.mode {
PassMode::Indirect { attrs, .. } => {
Ref(op_place_val) => {
let required_align = match attrs.pointee_align {
Some(pointee_align) => cmp::max(pointee_align, arg.layout.align.abi),
None => arg.layout.align.abi,
Expand All @@ -1493,15 +1468,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// to a higher-aligned alloca.
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
bx.typed_place_copy(scratch, op_place_val, op.layout);
(scratch.llval, scratch.align, true)
scratch.llval
} else {
(op_place_val.llval, op_place_val.align, true)
op_place_val.llval
}
}
_ => (op_place_val.llval, op_place_val.align, true),
},
ZeroSized => match arg.mode {
PassMode::Indirect { on_stack, .. } => {
ZeroSized => {
if on_stack {
// It doesn't seem like any target can have `byval` ZSTs, so this assert
// is here to replace a would-be untested codepath.
Expand All @@ -1511,59 +1483,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// a pointer for `repr(C)` structs even when empty, so get
// one from an `alloca` (which can be left uninitialized).
let scratch = PlaceRef::alloca(bx, arg.layout);
(scratch.val.llval, scratch.val.align, true)
scratch.val.llval
}
_ => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"),
},
};

if by_ref && !arg.is_indirect() {
// Have to load the argument, maybe while casting it.
if let PassMode::Cast { cast, pad_i32: _ } = &arg.mode {
// The ABI mandates that the value is passed as a different struct representation.
// Spill and reload it from the stack to convert from the Rust representation to
// the ABI representation.
let scratch_size = cast.size(bx);
let scratch_align = cast.align(bx);
// Note that the ABI type may be either larger or smaller than the Rust type,
// due to the presence or absence of trailing padding. For example:
// - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
// when passed by value, making it smaller.
// - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
// when passed by value, making it larger.
let copy_bytes = cmp::min(cast.unaligned_size(bx).bytes(), arg.layout.size.bytes());
// Allocate some scratch space...
let llscratch = bx.alloca(scratch_size, scratch_align);
bx.lifetime_start(llscratch, scratch_size);
// ...memcpy the value...
bx.memcpy(
llscratch,
scratch_align,
llval,
align,
bx.const_usize(copy_bytes),
MemFlags::empty(),
);
// ...and then load it with the ABI type.
let cast_ty = bx.cast_backend_type(cast);
llval = bx.load(cast_ty, llscratch, scratch_align);
bx.lifetime_end(llscratch, scratch_size);
} else {
// We can't use `PlaceRef::load` here because the argument
// may have a type we don't treat as immediate, but the ABI
// used for this call is passing it by-value. In that case,
// the load would just produce `OperandValue::Ref` instead
// of the `OperandValue::Immediate` we need for the call.
llval = bx.load(bx.backend_type(arg.layout), llval, align);
if let abi::Abi::Scalar(scalar) = arg.layout.abi {
if scalar.is_bool() {
PassMode::Cast { ref cast, .. } => cast.cast_rust_abi_to_other(bx, op),
_ => match op.val {
Immediate(_) | Pair(..) => op.immediate_or_packed_pair(bx),
Ref(op_place_val) => {
// We can't use `PlaceRef::load` here because the argument
// may have a type we don't treat as immediate, but the ABI
// used for this call is passing it by-value. In that case,
// the load would just produce `OperandValue::Ref` instead
// of the `OperandValue::Immediate` we need for the call.
let mut llval = bx.load(
bx.backend_type(arg.layout),
op_place_val.llval,
op_place_val.align,
);
if let abi::Abi::Scalar(scalar) = arg.layout.abi
&& scalar.is_bool()
{
bx.range_metadata(llval, WrappingRange { start: 0, end: 1 });
}
// We store bools as `i8` so we need to truncate to `i1`.
llval = bx.to_immediate(llval, arg.layout);
llval
}
// We store bools as `i8` so we need to truncate to `i1`.
llval = bx.to_immediate(llval, arg.layout);
}
}
ZeroSized => bug!("ZST {op:?} wasn't ignored, but was passed with abi {arg:?}"),
},
};

llargs.push(llval);
}
Expand Down
18 changes: 4 additions & 14 deletions compiler/rustc_codegen_ssa/src/mir/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,20 +229,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let layout = start_bx.layout_of(fx.monomorphize(decl.ty));
assert!(!layout.ty.has_erasable_regions());

if local == mir::RETURN_PLACE {
match fx.fn_abi.ret.mode {
PassMode::Indirect { .. } => {
debug!("alloc: {:?} (return place) -> place", local);
let llretptr = start_bx.get_param(0);
return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
}
PassMode::Cast { ref cast, .. } => {
debug!("alloc: {:?} (return place) -> place", local);
let size = cast.size(&start_bx);
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
}
_ => {}
};
if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() {
debug!("alloc: {:?} (return place) -> place", local);
let llretptr = start_bx.get_param(0);
return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
}

if memory_locals.contains(local) {
Expand Down
Loading
Loading