From 009280c5e312bdf11cd0e0e1b336bf374eed7b00 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 20 Mar 2024 14:38:32 +0100
Subject: [PATCH 1/2] Fix argument ABI for overaligned structs on ppc64le

When passing a 16 (or higher) aligned struct by value on ppc64le,
it needs to be passed as an array of `i128` rather than an array
of `i64`. This will force the use of an even starting register.

For the case of a 16 byte struct with alignment 16 it is important
that `[1 x i128]` is used instead of `i128` -- apparently, the
latter will get treated similarly to `[2 x i64]`, not exhibiting
the correct ABI. Add a `force_array` flag to `Uniform` to support
this.

The relevant clang code can be found here:
https://github.com/llvm/llvm-project/blob/fe2119a7b08b6e468b2a67768904ea85b1bf0a45/clang/lib/CodeGen/Targets/PPC.cpp#L878-L884
https://github.com/llvm/llvm-project/blob/fe2119a7b08b6e468b2a67768904ea85b1bf0a45/clang/lib/CodeGen/Targets/PPC.cpp#L780-L784

I think the corresponding psABI wording is this:

> Fixed size aggregates and unions passed by value are mapped to as
> many doublewords of the parameter save area as the value uses in
> memory. Aggregrates and unions are aligned according to their
> alignment requirements. This may result in doublewords being
> skipped for alignment.

In particular the last sentence.

Fixes https://github.com/rust-lang/rust/issues/122767.
---
 compiler/rustc_codegen_llvm/src/abi.rs        |  2 +-
 compiler/rustc_target/src/abi/call/aarch64.rs |  8 +-
 compiler/rustc_target/src/abi/call/arm.rs     | 10 +-
 compiler/rustc_target/src/abi/call/csky.rs    |  4 +-
 .../rustc_target/src/abi/call/loongarch.rs    |  7 +-
 compiler/rustc_target/src/abi/call/mips.rs    |  5 +-
 compiler/rustc_target/src/abi/call/mips64.rs  |  4 +-
 compiler/rustc_target/src/abi/call/mod.rs     |  5 +-
 compiler/rustc_target/src/abi/call/nvptx64.rs |  2 +-
 .../rustc_target/src/abi/call/powerpc64.rs    | 25 ++---
 compiler/rustc_target/src/abi/call/riscv.rs   |  7 +-
 compiler/rustc_target/src/abi/call/sparc.rs   |  5 +-
 compiler/rustc_target/src/abi/call/sparc64.rs |  4 +-
 compiler/rustc_target/src/abi/call/wasm.rs    |  4 +-
 tests/codegen/powerpc64le-struct-align-128.rs | 93 +++++++++++++++++++
 15 files changed, 152 insertions(+), 33 deletions(-)
 create mode 100644 tests/codegen/powerpc64le-struct-align-128.rs

diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index d2828669d438f..8b2330471f763 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -150,7 +150,7 @@ impl LlvmType for CastTarget {
         // Simplify to a single unit or an array if there's no prefix.
         // This produces the same layout, but using a simpler type.
         if self.prefix.iter().all(|x| x.is_none()) {
-            if rest_count == 1 {
+            if rest_count == 1 && !self.rest.force_array {
                 return rest_ll_unit;
             }
 
diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs
index f99f6a3b72164..bf9cf232d8140 100644
--- a/compiler/rustc_target/src/abi/call/aarch64.rs
+++ b/compiler/rustc_target/src/abi/call/aarch64.rs
@@ -31,7 +31,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size })
+        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
     })
 }
 
@@ -60,7 +60,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 128 {
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size });
+        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
         return;
     }
     ret.make_indirect();
@@ -100,9 +100,9 @@ where
     };
     if size.bits() <= 128 {
         if align.bits() == 128 {
-            arg.cast_to(Uniform { unit: Reg::i128(), total: size });
+            arg.cast_to(Uniform { unit: Reg::i128(), total: size, force_array: false });
         } else {
-            arg.cast_to(Uniform { unit: Reg::i64(), total: size });
+            arg.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
         }
         return;
     }
diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs
index 95f6691d42aeb..177a16bafc837 100644
--- a/compiler/rustc_target/src/abi/call/arm.rs
+++ b/compiler/rustc_target/src/abi/call/arm.rs
@@ -21,7 +21,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size })
+        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
     })
 }
 
@@ -49,7 +49,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 32 {
-        ret.cast_to(Uniform { unit: Reg::i32(), total: size });
+        ret.cast_to(Uniform { unit: Reg::i32(), total: size, force_array: false });
         return;
     }
     ret.make_indirect();
@@ -78,7 +78,11 @@ where
 
     let align = arg.layout.align.abi.bytes();
     let total = arg.layout.size;
-    arg.cast_to(Uniform { unit: if align <= 4 { Reg::i32() } else { Reg::i64() }, total });
+    arg.cast_to(Uniform {
+        unit: if align <= 4 { Reg::i32() } else { Reg::i64() },
+        total,
+        force_array: false,
+    });
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs
index 8b4328db52ebb..dae428e3538dc 100644
--- a/compiler/rustc_target/src/abi/call/csky.rs
+++ b/compiler/rustc_target/src/abi/call/csky.rs
@@ -18,7 +18,7 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
         if total.bits() > 64 {
             arg.make_indirect();
         } else if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total });
+            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
         } else {
             arg.cast_to(Reg::i32());
         }
@@ -38,7 +38,7 @@ fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
     if arg.layout.is_aggregate() {
         let total = arg.layout.size;
         if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total });
+            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
         } else {
             arg.cast_to(Reg::i32());
         }
diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs
index 35d4b331cb4d3..3b8a3ba66ba8b 100644
--- a/compiler/rustc_target/src/abi/call/loongarch.rs
+++ b/compiler/rustc_target/src/abi/call/loongarch.rs
@@ -195,7 +195,11 @@ where
         if total.bits() <= xlen {
             arg.cast_to(xlen_reg);
         } else {
-            arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) });
+            arg.cast_to(Uniform {
+                unit: xlen_reg,
+                total: Size::from_bits(xlen * 2),
+                force_array: false,
+            });
         }
         return false;
     }
@@ -281,6 +285,7 @@ fn classify_arg<'a, Ty, C>(
             arg.cast_to(Uniform {
                 unit: if align_regs { double_xlen_reg } else { xlen_reg },
                 total: Size::from_bits(xlen * 2),
+                force_array: false,
             });
         }
         if align_regs && is_vararg {
diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs
index 57ccfe2152bdf..054c127181ad8 100644
--- a/compiler/rustc_target/src/abi/call/mips.rs
+++ b/compiler/rustc_target/src/abi/call/mips.rs
@@ -27,7 +27,10 @@ where
 
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
-        arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32);
+        arg.cast_to_and_pad_i32(
+            Uniform { unit: Reg::i32(), total: size, force_array: false },
+            pad_i32,
+        );
     } else {
         arg.extend_integer_width_to(32);
     }
diff --git a/compiler/rustc_target/src/abi/call/mips64.rs b/compiler/rustc_target/src/abi/call/mips64.rs
index 2700f67b2096b..69e73beca9a6f 100644
--- a/compiler/rustc_target/src/abi/call/mips64.rs
+++ b/compiler/rustc_target/src/abi/call/mips64.rs
@@ -68,7 +68,7 @@ where
         }
 
         // Cast to a uniform int structure
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size });
+        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
     } else {
         ret.make_indirect();
     }
@@ -139,7 +139,7 @@ where
     let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
     arg.cast_to(CastTarget {
         prefix,
-        rest: Uniform { unit: Reg::i64(), total: rest_size },
+        rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
         attrs: ArgAttributes {
             regular: ArgAttribute::default(),
             arg_ext: ArgExtension::None,
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index cdd3f0afd79a6..022ea44cc934a 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -255,11 +255,14 @@ pub struct Uniform {
     ///   for 64-bit integers with a total size of 20 bytes. When the argument is actually passed,
     ///   this size will be rounded up to the nearest multiple of `unit.size`.
     pub total: Size,
+
+    /// Force the use of an array, even if there is only a single element.
+    pub force_array: bool,
 }
 
 impl From<Reg> for Uniform {
     fn from(unit: Reg) -> Uniform {
-        Uniform { unit, total: unit.size }
+        Uniform { unit, total: unit.size, force_array: false }
     }
 }
 
diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs
index 5c040ce9c3b10..e7525c760153b 100644
--- a/compiler/rustc_target/src/abi/call/nvptx64.rs
+++ b/compiler/rustc_target/src/abi/call/nvptx64.rs
@@ -35,7 +35,7 @@ where
             16 => Reg::i128(),
             _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"),
         };
-        arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes) });
+        arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes), force_array: false });
     } else {
         // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271.
         arg.make_direct_deprecated();
diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs
index 2d41f77e50e1e..0b2bd8c90d791 100644
--- a/compiler/rustc_target/src/abi/call/powerpc64.rs
+++ b/compiler/rustc_target/src/abi/call/powerpc64.rs
@@ -2,7 +2,7 @@
 // Alignment of 128 bit types is not currently handled, this will
 // need to be fixed when PowerPC vector support is added.
 
-use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind, Uniform};
+use crate::abi::call::{Align, ArgAbi, FnAbi, Reg, RegKind, Uniform};
 use crate::abi::{Endian, HasDataLayout, TyAbiInterface};
 use crate::spec::HasTargetSpec;
 
@@ -37,7 +37,7 @@ where
             RegKind::Vector => arg.layout.size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: arg.layout.size })
+        valid_unit.then_some(Uniform { unit, total: arg.layout.size, force_array: false })
     })
 }
 
@@ -81,7 +81,7 @@ where
             Reg::i64()
         };
 
-        ret.cast_to(Uniform { unit, total: size });
+        ret.cast_to(Uniform { unit, total: size, force_array: false });
         return;
     }
 
@@ -108,18 +108,21 @@ where
     }
 
     let size = arg.layout.size;
-    let (unit, total) = if size.bits() <= 64 {
+    if size.bits() <= 64 {
         // Aggregates smaller than a doubleword should appear in
         // the least-significant bits of the parameter doubleword.
-        (Reg { kind: RegKind::Integer, size }, size)
+        arg.cast_to(Reg { kind: RegKind::Integer, size })
     } else {
-        // Aggregates larger than a doubleword should be padded
-        // at the tail to fill out a whole number of doublewords.
-        let reg_i64 = Reg::i64();
-        (reg_i64, size.align_to(reg_i64.align(cx)))
+        // Aggregates larger than i64 should be padded at the tail to fill out a whole number
+        // of i64s or i128s, depending on the aggregate alignment. Always use an array for
+        // this, even if there is only a single element.
+        let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() };
+        arg.cast_to(Uniform {
+            unit: reg,
+            total: size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
+            force_array: true,
+        })
     };
-
-    arg.cast_to(Uniform { unit, total });
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs
index 6a38496dc5761..0dad35c8aab5a 100644
--- a/compiler/rustc_target/src/abi/call/riscv.rs
+++ b/compiler/rustc_target/src/abi/call/riscv.rs
@@ -201,7 +201,11 @@ where
         if total.bits() <= xlen {
             arg.cast_to(xlen_reg);
         } else {
-            arg.cast_to(Uniform { unit: xlen_reg, total: Size::from_bits(xlen * 2) });
+            arg.cast_to(Uniform {
+                unit: xlen_reg,
+                total: Size::from_bits(xlen * 2),
+                force_array: false,
+            });
         }
         return false;
     }
@@ -287,6 +291,7 @@ fn classify_arg<'a, Ty, C>(
             arg.cast_to(Uniform {
                 unit: if align_regs { double_xlen_reg } else { xlen_reg },
                 total: Size::from_bits(xlen * 2),
+                force_array: false,
             });
         }
         if align_regs && is_vararg {
diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs
index 57ccfe2152bdf..054c127181ad8 100644
--- a/compiler/rustc_target/src/abi/call/sparc.rs
+++ b/compiler/rustc_target/src/abi/call/sparc.rs
@@ -27,7 +27,10 @@ where
 
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
-        arg.cast_to_and_pad_i32(Uniform { unit: Reg::i32(), total: size }, pad_i32);
+        arg.cast_to_and_pad_i32(
+            Uniform { unit: Reg::i32(), total: size, force_array: false },
+            pad_i32,
+        );
     } else {
         arg.extend_integer_width_to(32);
     }
diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs
index cbed5b4afc134..a36b691468e4a 100644
--- a/compiler/rustc_target/src/abi/call/sparc64.rs
+++ b/compiler/rustc_target/src/abi/call/sparc64.rs
@@ -192,7 +192,7 @@ where
 
                 arg.cast_to(CastTarget {
                     prefix: data.prefix,
-                    rest: Uniform { unit: Reg::i64(), total: rest_size },
+                    rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
                     attrs: ArgAttributes {
                         regular: data.arg_attribute,
                         arg_ext: ArgExtension::None,
@@ -205,7 +205,7 @@ where
         }
     }
 
-    arg.cast_to(Uniform { unit: Reg::i64(), total });
+    arg.cast_to(Uniform { unit: Reg::i64(), total, force_array: false });
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs
index a7a2b314a94f0..a773fb1e814e0 100644
--- a/compiler/rustc_target/src/abi/call/wasm.rs
+++ b/compiler/rustc_target/src/abi/call/wasm.rs
@@ -1,4 +1,4 @@
-use crate::abi::call::{ArgAbi, FnAbi, Uniform};
+use crate::abi::call::{ArgAbi, FnAbi};
 use crate::abi::{HasDataLayout, TyAbiInterface};
 
 fn unwrap_trivial_aggregate<'a, Ty, C>(cx: &C, val: &mut ArgAbi<'a, Ty>) -> bool
@@ -10,7 +10,7 @@ where
         if let Some(unit) = val.layout.homogeneous_aggregate(cx).ok().and_then(|ha| ha.unit()) {
             let size = val.layout.size;
             if unit.size == size {
-                val.cast_to(Uniform { unit, total: size });
+                val.cast_to(unit);
                 return true;
             }
         }
diff --git a/tests/codegen/powerpc64le-struct-align-128.rs b/tests/codegen/powerpc64le-struct-align-128.rs
new file mode 100644
index 0000000000000..0096c6d3138b8
--- /dev/null
+++ b/tests/codegen/powerpc64le-struct-align-128.rs
@@ -0,0 +1,93 @@
+// Test that structs aligned to 128 bits are passed with the correct ABI on powerpc64le.
+// This is similar to aarch64-struct-align-128.rs, but for ppc.
+
+//@ compile-flags: --target powerpc64le-unknown-linux-gnu
+//@ needs-llvm-components: powerpc
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+#[lang="sized"]
+trait Sized { }
+#[lang="freeze"]
+trait Freeze { }
+#[lang="copy"]
+trait Copy { }
+
+#[repr(C)]
+pub struct Align8 {
+    pub a: u64,
+    pub b: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent8 {
+    a: Align8
+}
+
+#[repr(C)]
+pub struct Wrapped8 {
+    a: Align8,
+}
+
+extern "C" {
+    // CHECK: declare void @test_8([2 x i64], [2 x i64], [2 x i64])
+    fn test_8(a: Align8, b: Transparent8, c: Wrapped8);
+}
+
+#[repr(C)]
+#[repr(align(16))]
+pub struct Align16 {
+    pub a: u64,
+    pub b: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent16 {
+    a: Align16
+}
+
+#[repr(C)]
+pub struct Wrapped16 {
+    pub a: Align16,
+}
+
+extern "C" {
+    // It's important that this produces [1 x i128]  rather than just i128!
+    // CHECK: declare void @test_16([1 x i128], [1 x i128], [1 x i128])
+    fn test_16(a: Align16, b: Transparent16, c: Wrapped16);
+}
+
+#[repr(C)]
+#[repr(align(32))]
+pub struct Align32 {
+    pub a: u64,
+    pub b: u64,
+    pub c: u64,
+}
+
+#[repr(transparent)]
+pub struct Transparent32 {
+    a: Align32
+}
+
+#[repr(C)]
+pub struct Wrapped32 {
+    pub a: Align32,
+}
+
+extern "C" {
+    // CHECK: declare void @test_32([2 x i128], [2 x i128], [2 x i128])
+    fn test_32(a: Align32, b: Transparent32, c: Wrapped32);
+}
+
+pub unsafe fn main(
+    a1: Align8, a2: Transparent8, a3: Wrapped8,
+    b1: Align16, b2: Transparent16, b3: Wrapped16,
+    c1: Align32, c2: Transparent32, c3: Wrapped32,
+) {
+    test_8(a1, a2, a3);
+    test_16(b1, b2, b3);
+    test_32(c1, c2, c3);
+}

From 1b7342b4117e321012cbe49df5ae859d52031f2b Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 21 Mar 2024 16:10:23 +0100
Subject: [PATCH 2/2] force_array -> is_consecutive

The actual ABI implication here is that in some cases the values
are required to be "consecutive", i.e. must either all be passed
in registers or all on stack (without padding).

Adjust the code to either use Uniform::new() or Uniform::consecutive()
depending on which behavior is needed.

Then, when lowering this in LLVM, skip the [1 x i128] to i128
simplification if is_consecutive is set. i128 is the only case
I'm aware of where this is problematic right now. If we find
other cases, we can extend this (either based on target information
or possibly just by not simplifying for is_consecutive entirely).
---
 compiler/rustc_codegen_llvm/src/abi.rs        |  5 ++++-
 compiler/rustc_target/src/abi/call/aarch64.rs |  8 ++++----
 compiler/rustc_target/src/abi/call/arm.rs     | 10 +++-------
 compiler/rustc_target/src/abi/call/csky.rs    |  4 ++--
 .../rustc_target/src/abi/call/loongarch.rs    | 15 +++++---------
 compiler/rustc_target/src/abi/call/mips.rs    |  5 +----
 compiler/rustc_target/src/abi/call/mips64.rs  |  4 ++--
 compiler/rustc_target/src/abi/call/mod.rs     | 20 ++++++++++++++++---
 compiler/rustc_target/src/abi/call/nvptx64.rs |  2 +-
 .../rustc_target/src/abi/call/powerpc64.rs    | 13 ++++++------
 compiler/rustc_target/src/abi/call/riscv.rs   | 15 +++++---------
 compiler/rustc_target/src/abi/call/sparc.rs   |  5 +----
 compiler/rustc_target/src/abi/call/sparc64.rs |  4 ++--
 13 files changed, 53 insertions(+), 57 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index 8b2330471f763..f918facc86db1 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -150,7 +150,10 @@ impl LlvmType for CastTarget {
         // Simplify to a single unit or an array if there's no prefix.
         // This produces the same layout, but using a simpler type.
         if self.prefix.iter().all(|x| x.is_none()) {
-            if rest_count == 1 && !self.rest.force_array {
+            // We can't do this if is_consecutive is set and the unit would get
+            // split on the target. Currently, this is only relevant for i128
+            // registers.
+            if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) {
                 return rest_ll_unit;
             }
 
diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs
index bf9cf232d8140..04020d13f22f9 100644
--- a/compiler/rustc_target/src/abi/call/aarch64.rs
+++ b/compiler/rustc_target/src/abi/call/aarch64.rs
@@ -31,7 +31,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, size))
     })
 }
 
@@ -60,7 +60,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 128 {
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i64(), size));
         return;
     }
     ret.make_indirect();
@@ -100,9 +100,9 @@ where
     };
     if size.bits() <= 128 {
         if align.bits() == 128 {
-            arg.cast_to(Uniform { unit: Reg::i128(), total: size, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i128(), size));
         } else {
-            arg.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i64(), size));
         }
         return;
     }
diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs
index 177a16bafc837..9371e1b395865 100644
--- a/compiler/rustc_target/src/abi/call/arm.rs
+++ b/compiler/rustc_target/src/abi/call/arm.rs
@@ -21,7 +21,7 @@ where
             RegKind::Vector => size.bits() == 64 || size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, size))
     })
 }
 
@@ -49,7 +49,7 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 32 {
-        ret.cast_to(Uniform { unit: Reg::i32(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i32(), size));
         return;
     }
     ret.make_indirect();
@@ -78,11 +78,7 @@ where
 
     let align = arg.layout.align.abi.bytes();
     let total = arg.layout.size;
-    arg.cast_to(Uniform {
-        unit: if align <= 4 { Reg::i32() } else { Reg::i64() },
-        total,
-        force_array: false,
-    });
+    arg.cast_to(Uniform::consecutive(if align <= 4 { Reg::i32() } else { Reg::i64() }, total));
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs
index dae428e3538dc..7951f28beeaff 100644
--- a/compiler/rustc_target/src/abi/call/csky.rs
+++ b/compiler/rustc_target/src/abi/call/csky.rs
@@ -18,7 +18,7 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
         if total.bits() > 64 {
             arg.make_indirect();
         } else if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i32(), total));
         } else {
             arg.cast_to(Reg::i32());
         }
@@ -38,7 +38,7 @@ fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
     if arg.layout.is_aggregate() {
         let total = arg.layout.size;
         if total.bits() > 32 {
-            arg.cast_to(Uniform { unit: Reg::i32(), total, force_array: false });
+            arg.cast_to(Uniform::new(Reg::i32(), total));
         } else {
             arg.cast_to(Reg::i32());
         }
diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs
index 3b8a3ba66ba8b..943b12a9fbfcc 100644
--- a/compiler/rustc_target/src/abi/call/loongarch.rs
+++ b/compiler/rustc_target/src/abi/call/loongarch.rs
@@ -195,11 +195,7 @@ where
         if total.bits() <= xlen {
             arg.cast_to(xlen_reg);
         } else {
-            arg.cast_to(Uniform {
-                unit: xlen_reg,
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2)));
         }
         return false;
     }
@@ -282,11 +278,10 @@ fn classify_arg<'a, Ty, C>(
     if total.bits() > xlen {
         let align_regs = align > xlen;
         if is_loongarch_aggregate(arg) {
-            arg.cast_to(Uniform {
-                unit: if align_regs { double_xlen_reg } else { xlen_reg },
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(
+                if align_regs { double_xlen_reg } else { xlen_reg },
+                Size::from_bits(xlen * 2),
+            ));
         }
         if align_regs && is_vararg {
             *avail_gprs -= *avail_gprs % 2;
diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs
index 054c127181ad8..0e5a7f37a094c 100644
--- a/compiler/rustc_target/src/abi/call/mips.rs
+++ b/compiler/rustc_target/src/abi/call/mips.rs
@@ -27,10 +27,7 @@ where
 
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
-        arg.cast_to_and_pad_i32(
-            Uniform { unit: Reg::i32(), total: size, force_array: false },
-            pad_i32,
-        );
+        arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
     } else {
         arg.extend_integer_width_to(32);
     }
diff --git a/compiler/rustc_target/src/abi/call/mips64.rs b/compiler/rustc_target/src/abi/call/mips64.rs
index 69e73beca9a6f..b2a2c34b980f6 100644
--- a/compiler/rustc_target/src/abi/call/mips64.rs
+++ b/compiler/rustc_target/src/abi/call/mips64.rs
@@ -68,7 +68,7 @@ where
         }
 
         // Cast to a uniform int structure
-        ret.cast_to(Uniform { unit: Reg::i64(), total: size, force_array: false });
+        ret.cast_to(Uniform::new(Reg::i64(), size));
     } else {
         ret.make_indirect();
     }
@@ -139,7 +139,7 @@ where
     let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
     arg.cast_to(CastTarget {
         prefix,
-        rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
+        rest: Uniform::new(Reg::i64(), rest_size),
         attrs: ArgAttributes {
             regular: ArgAttribute::default(),
             arg_ext: ArgExtension::None,
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 022ea44cc934a..4502df339d1d8 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -256,13 +256,15 @@ pub struct Uniform {
     ///   this size will be rounded up to the nearest multiple of `unit.size`.
     pub total: Size,
 
-    /// Force the use of an array, even if there is only a single element.
-    pub force_array: bool,
+    /// Indicate that the argument is consecutive, in the sense that either all values need to be
+    /// passed in register, or all on the stack. If they are passed on the stack, there should be
+    /// no additional padding between elements.
+    pub is_consecutive: bool,
 }
 
 impl From<Reg> for Uniform {
     fn from(unit: Reg) -> Uniform {
-        Uniform { unit, total: unit.size, force_array: false }
+        Uniform { unit, total: unit.size, is_consecutive: false }
     }
 }
 
@@ -270,6 +272,18 @@ impl Uniform {
     pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
         self.unit.align(cx)
     }
+
+    /// Pass using one or more values of the given type, without requiring them to be consecutive.
+    /// That is, some values may be passed in register and some on the stack.
+    pub fn new(unit: Reg, total: Size) -> Self {
+        Uniform { unit, total, is_consecutive: false }
+    }
+
+    /// Pass using one or more consecutive values of the given type. Either all values will be
+    /// passed in registers, or all on the stack.
+    pub fn consecutive(unit: Reg, total: Size) -> Self {
+        Uniform { unit, total, is_consecutive: true }
+    }
 }
 
 /// Describes the type used for `PassMode::Cast`.
diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs
index e7525c760153b..f85fa2419f0f6 100644
--- a/compiler/rustc_target/src/abi/call/nvptx64.rs
+++ b/compiler/rustc_target/src/abi/call/nvptx64.rs
@@ -35,7 +35,7 @@ where
             16 => Reg::i128(),
             _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"),
         };
-        arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes), force_array: false });
+        arg.cast_to(Uniform::new(unit, Size::from_bytes(2 * align_bytes)));
     } else {
         // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271.
         arg.make_direct_deprecated();
diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs
index 0b2bd8c90d791..11a6cb52babc9 100644
--- a/compiler/rustc_target/src/abi/call/powerpc64.rs
+++ b/compiler/rustc_target/src/abi/call/powerpc64.rs
@@ -37,7 +37,7 @@ where
             RegKind::Vector => arg.layout.size.bits() == 128,
         };
 
-        valid_unit.then_some(Uniform { unit, total: arg.layout.size, force_array: false })
+        valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size))
     })
 }
 
@@ -81,7 +81,7 @@ where
             Reg::i64()
         };
 
-        ret.cast_to(Uniform { unit, total: size, force_array: false });
+        ret.cast_to(Uniform::new(unit, size));
         return;
     }
 
@@ -117,11 +117,10 @@ where
         // of i64s or i128s, depending on the aggregate alignment. Always use an array for
         // this, even if there is only a single element.
         let reg = if arg.layout.align.abi.bytes() > 8 { Reg::i128() } else { Reg::i64() };
-        arg.cast_to(Uniform {
-            unit: reg,
-            total: size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
-            force_array: true,
-        })
+        arg.cast_to(Uniform::consecutive(
+            reg,
+            size.align_to(Align::from_bytes(reg.size.bytes()).unwrap()),
+        ))
     };
 }
 
diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs
index 0dad35c8aab5a..5d4b3a9d245a4 100644
--- a/compiler/rustc_target/src/abi/call/riscv.rs
+++ b/compiler/rustc_target/src/abi/call/riscv.rs
@@ -201,11 +201,7 @@ where
         if total.bits() <= xlen {
             arg.cast_to(xlen_reg);
         } else {
-            arg.cast_to(Uniform {
-                unit: xlen_reg,
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(xlen_reg, Size::from_bits(xlen * 2)));
         }
         return false;
     }
@@ -288,11 +284,10 @@ fn classify_arg<'a, Ty, C>(
     if total.bits() > xlen {
         let align_regs = align > xlen;
         if is_riscv_aggregate(arg) {
-            arg.cast_to(Uniform {
-                unit: if align_regs { double_xlen_reg } else { xlen_reg },
-                total: Size::from_bits(xlen * 2),
-                force_array: false,
-            });
+            arg.cast_to(Uniform::new(
+                if align_regs { double_xlen_reg } else { xlen_reg },
+                Size::from_bits(xlen * 2),
+            ));
         }
         if align_regs && is_vararg {
             *avail_gprs -= *avail_gprs % 2;
diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs
index 054c127181ad8..0e5a7f37a094c 100644
--- a/compiler/rustc_target/src/abi/call/sparc.rs
+++ b/compiler/rustc_target/src/abi/call/sparc.rs
@@ -27,10 +27,7 @@ where
 
     if arg.layout.is_aggregate() {
         let pad_i32 = !offset.is_aligned(align);
-        arg.cast_to_and_pad_i32(
-            Uniform { unit: Reg::i32(), total: size, force_array: false },
-            pad_i32,
-        );
+        arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
     } else {
         arg.extend_integer_width_to(32);
     }
diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs
index a36b691468e4a..acdcd5cc0d482 100644
--- a/compiler/rustc_target/src/abi/call/sparc64.rs
+++ b/compiler/rustc_target/src/abi/call/sparc64.rs
@@ -192,7 +192,7 @@ where
 
                 arg.cast_to(CastTarget {
                     prefix: data.prefix,
-                    rest: Uniform { unit: Reg::i64(), total: rest_size, force_array: false },
+                    rest: Uniform::new(Reg::i64(), rest_size),
                     attrs: ArgAttributes {
                         regular: data.arg_attribute,
                         arg_ext: ArgExtension::None,
@@ -205,7 +205,7 @@ where
         }
     }
 
-    arg.cast_to(Uniform { unit: Reg::i64(), total, force_array: false });
+    arg.cast_to(Uniform::new(Reg::i64(), total));
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)