From 31d23c436aabdb6509f39f09681011176add3da1 Mon Sep 17 00:00:00 2001
From: David Rheinsberg <david@readahead.eu>
Date: Mon, 24 Jul 2023 14:11:26 +0200
Subject: [PATCH] compiler: allow transmute of ZST arrays with generics

Extend the `SizeSkeleton` evaluator to shortcut zero-sized arrays, thus
considering `[T; 0]` to have a compile-time fixed-size of 0.

The existing evaluator already deals with generic arrays under the
feature-guard `transmute_const_generics`. However, it merely allows
comparing fixed-size types with fixed-size types, and generic types with
generic types. For generic types, it merely compares whether their
arguments match (ordering them first). Even if their exact sizes are not
known at compile time, it can ensure that they will eventually be the
same.

This patch extends this by shortcutting the size-evaluation of zero
sized arrays and thus allowing size comparisons of `()` with `[T; 0]`,
where one contains generics and the other does not.

This code is guarded by `transmute_const_generics` (#109929), even
though it is unclear whether it should be. However, this assumes that a
separate stabilization PR is required to move this out of the feature
guard.

Initially reported in #98104.
---
 compiler/rustc_middle/src/ty/layout.rs       |  7 +++-
 tests/ui/transmute/transmute-zst-generics.rs | 34 ++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/transmute/transmute-zst-generics.rs

diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 11fd73c909417..8d9f263189745 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -345,11 +345,16 @@ impl<'tcx> SizeSkeleton<'tcx> {
             ty::Array(inner, len)
                 if len.ty() == tcx.types.usize && tcx.features().transmute_generic_consts =>
             {
+                let len_eval = len.try_eval_target_usize(tcx, param_env);
+                if len_eval == Some(0) {
+                    return Ok(SizeSkeleton::Known(Size::from_bytes(0)));
+                }
+
                 match SizeSkeleton::compute(inner, tcx, param_env)? {
                     // This may succeed because the multiplication of two types may overflow
                     // but a single size of a nested array will not.
                     SizeSkeleton::Known(s) => {
-                        if let Some(c) = len.try_eval_target_usize(tcx, param_env) {
+                        if let Some(c) = len_eval {
                             let size = s
                                 .bytes()
                                 .checked_mul(c)
diff --git a/tests/ui/transmute/transmute-zst-generics.rs b/tests/ui/transmute/transmute-zst-generics.rs
new file mode 100644
index 0000000000000..9aeb21923eac6
--- /dev/null
+++ b/tests/ui/transmute/transmute-zst-generics.rs
@@ -0,0 +1,34 @@
+//@ run-pass
+
+// Transmuting to/from ZSTs that contain generics.
+
+#![feature(transmute_generic_consts)]
+
+// Verify non-generic ZST -> generic ZST transmute
+unsafe fn cast_zst0<T>(from: ()) -> [T; 0] {
+    ::std::mem::transmute::<(), [T; 0]>(from)
+}
+
+// Verify generic ZST -> non-generic ZST transmute
+unsafe fn cast_zst1<T>(from: [T; 0]) -> () {
+    ::std::mem::transmute::<[T; 0], ()>(from)
+}
+
+// Verify transmute with generic compound types
+unsafe fn cast_zst2<T>(from: ()) -> [(T, T); 0] {
+    ::std::mem::transmute::<(), [(T, T); 0]>(from)
+}
+
+// Verify transmute with ZST propagation through arrays
+unsafe fn cast_zst3<T>(from: ()) -> [[T; 0]; 8] {
+    ::std::mem::transmute::<(), [[T; 0]; 8]>(from)
+}
+
+pub fn main() {
+    unsafe {
+        let _: [u32; 0] = cast_zst0(());
+        let _ = cast_zst1::<u32>([]);
+        let _: [(u32, u32); 0] = cast_zst2(());
+        let _: [[u32; 0]; 8] = cast_zst3(());
+    };
+}