Auto merge of #108106 - the8472:layout-opt, r=wesleywiser

Improve niche placement by trying two strategies and picking the better result Fixes #104807 Fixes #105371 Determining which sort order is better requires calculating the struct size (so we can calculate the niche offset). But that in turn depends on the field order, so happens after sorting. So the simple way to solve that is to run the whole thing twice and pick the better result. 1st commit is just code motion, the meat is in the later ones.
rust-lang · Apr 29, 2023 · f229949 · f229949
2 parents af2c7e0 + 61fb5a9
commit f229949
Show file tree

Hide file tree

Showing 12 changed files with 511 additions and 298 deletions.
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
diff --git a/tests/codegen/issues/issue-103840.rs b/tests/codegen/issues/issue-103840.rs
@@ -1,4 +1,5 @@
 // compile-flags: -O
+// min-llvm-version: 16.0
 #![crate_type = "lib"]
 
 pub fn foo(t: &mut Vec<usize>) {

diff --git a/tests/codegen/issues/issue-105386-ub-in-debuginfo.rs b/tests/codegen/issues/issue-105386-ub-in-debuginfo.rs
@@ -19,4 +19,5 @@ pub fn outer_function(x: S, y: S) -> usize {
 // CHECK-NOT: [[ptr_tmp:%.*]] = getelementptr inbounds %"[closure@{{.*.rs}}:9:23: 9:25]", ptr [[spill]]
 // CHECK-NOT: [[load:%.*]] = load ptr, ptr
 // CHECK: call void @llvm.lifetime.start{{.*}}({{.*}}, ptr [[spill]])
-// CHECK: call void @llvm.memcpy{{.*}}(ptr {{align .*}} [[spill]], ptr {{align .*}} %x
+// CHECK: [[inner:%.*]] = getelementptr inbounds %"{{.*}}", ptr [[spill]]
+// CHECK: call void @llvm.memcpy{{.*}}(ptr {{align .*}} [[inner]], ptr {{align .*}} %x
diff --git a/tests/codegen/issues/issue-86106.rs b/tests/codegen/issues/issue-86106.rs
@@ -1,4 +1,5 @@
 // min-llvm-version: 15.0
+// only-64bit llvm appears to use stores instead of memset on 32bit
 // compile-flags: -C opt-level=3 -Z merge-functions=disabled
 
 // The below two functions ensure that both `String::new()` and `"".to_string()`
@@ -9,25 +10,19 @@
 // CHECK-LABEL: define void @string_new
 #[no_mangle]
 pub fn string_new() -> String {
-    // CHECK-NOT: load i8
-    // CHECK: store i{{32|64}}
+    // CHECK: store ptr inttoptr
     // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store ptr
-    // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store i{{32|64}}
+    // CHECK-NEXT: call void @llvm.memset
     // CHECK-NEXT: ret void
     String::new()
 }
 
 // CHECK-LABEL: define void @empty_to_string
 #[no_mangle]
 pub fn empty_to_string() -> String {
-    // CHECK-NOT: load i8
-    // CHECK: store i{{32|64}}
-    // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store ptr
+    // CHECK: store ptr inttoptr
     // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store i{{32|64}}
+    // CHECK-NEXT: call void @llvm.memset
     // CHECK-NEXT: ret void
     "".to_string()
 }
@@ -38,25 +33,19 @@ pub fn empty_to_string() -> String {
 // CHECK-LABEL: @empty_vec
 #[no_mangle]
 pub fn empty_vec() -> Vec<u8> {
-    // CHECK: store i{{32|64}}
-    // CHECK-NOT: load i8
+    // CHECK: store ptr inttoptr
     // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store ptr
-    // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store i{{32|64}}
+    // CHECK-NEXT: call void @llvm.memset
     // CHECK-NEXT: ret void
     vec![]
 }
 
 // CHECK-LABEL: @empty_vec_clone
 #[no_mangle]
 pub fn empty_vec_clone() -> Vec<u8> {
-    // CHECK: store i{{32|64}}
-    // CHECK-NOT: load i8
-    // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store ptr
+    // CHECK: store ptr inttoptr
     // CHECK-NEXT: getelementptr
-    // CHECK-NEXT: store i{{32|64}}
+    // CHECK-NEXT: call void @llvm.memset
     // CHECK-NEXT: ret void
     vec![].clone()
 }
diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout
@@ -2,38 +2,34 @@ print-type-size type: `[async fn body@$DIR/async-awaiting-fut.rs:21:21: 24:2]`:
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 0 bytes
 print-type-size     variant `Suspend0`: 3077 bytes
-print-type-size         local `.__awaitee`: 3077 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         local `.__awaitee`: 3077 bytes
 print-type-size     variant `Returned`: 0 bytes
 print-type-size     variant `Panicked`: 0 bytes
 print-type-size type: `[async fn body@$DIR/async-awaiting-fut.rs:10:64: 19:2]`: 3077 bytes, alignment: 1 bytes
 print-type-size     discriminant: 1 bytes
-print-type-size     variant `Unresumed`: 2051 bytes
-print-type-size         padding: 1026 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size     variant `Unresumed`: 1025 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
 print-type-size     variant `Suspend0`: 2052 bytes
-print-type-size         local `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
-print-type-size         local `..generator_field4`: 1 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
 print-type-size         padding: 1 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size         local `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size         local `..generator_field4`: 1 bytes
 print-type-size         local `.__awaitee`: 1 bytes
 print-type-size     variant `Suspend1`: 3076 bytes
-print-type-size         padding: 1024 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         padding: 1026 bytes
 print-type-size         local `..generator_field4`: 1 bytes, alignment: 1 bytes
-print-type-size         padding: 1 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
 print-type-size         local `.__awaitee`: 1025 bytes
 print-type-size     variant `Suspend2`: 2052 bytes
-print-type-size         local `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
-print-type-size         local `..generator_field4`: 1 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
 print-type-size         padding: 1 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size         local `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size         local `..generator_field4`: 1 bytes
 print-type-size         local `.__awaitee`: 1 bytes
-print-type-size     variant `Returned`: 2051 bytes
-print-type-size         padding: 1026 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
-print-type-size     variant `Panicked`: 2051 bytes
-print-type-size         padding: 1026 bytes
-print-type-size         upvar `.fut`: 1025 bytes, alignment: 1 bytes
+print-type-size     variant `Returned`: 1025 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size     variant `Panicked`: 1025 bytes
+print-type-size         upvar `.fut`: 1025 bytes, offset: 0 bytes, alignment: 1 bytes
 print-type-size type: `std::mem::ManuallyDrop<[async fn body@$DIR/async-awaiting-fut.rs:10:64: 19:2]>`: 3077 bytes, alignment: 1 bytes
 print-type-size     field `.value`: 3077 bytes
 print-type-size type: `std::mem::MaybeUninit<[async fn body@$DIR/async-awaiting-fut.rs:10:64: 19:2]>`: 3077 bytes, alignment: 1 bytes
@@ -43,11 +39,11 @@ print-type-size         field `.value`: 3077 bytes
 print-type-size type: `[async fn body@$DIR/async-awaiting-fut.rs:8:35: 8:37]`: 1025 bytes, alignment: 1 bytes
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 1024 bytes
-print-type-size         upvar `.arg`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.arg`: 1024 bytes
 print-type-size     variant `Returned`: 1024 bytes
-print-type-size         upvar `.arg`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.arg`: 1024 bytes
 print-type-size     variant `Panicked`: 1024 bytes
-print-type-size         upvar `.arg`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.arg`: 1024 bytes
 print-type-size type: `std::mem::ManuallyDrop<[async fn body@$DIR/async-awaiting-fut.rs:8:35: 8:37]>`: 1025 bytes, alignment: 1 bytes
 print-type-size     field `.value`: 1025 bytes
 print-type-size type: `std::mem::MaybeUninit<[async fn body@$DIR/async-awaiting-fut.rs:8:35: 8:37]>`: 1025 bytes, alignment: 1 bytes

diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.stdout
@@ -2,20 +2,20 @@ print-type-size type: `[async fn body@$DIR/large-arg.rs:6:21: 8:2]`: 3076 bytes,
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 0 bytes
 print-type-size     variant `Suspend0`: 3075 bytes
-print-type-size         local `.__awaitee`: 3075 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         local `.__awaitee`: 3075 bytes
 print-type-size     variant `Returned`: 0 bytes
 print-type-size     variant `Panicked`: 0 bytes
 print-type-size type: `[async fn body@$DIR/large-arg.rs:10:30: 12:2]`: 3075 bytes, alignment: 1 bytes
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Suspend0`: 3074 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size         local `.__awaitee`: 2050 bytes
 print-type-size     variant `Returned`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Panicked`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size type: `std::mem::ManuallyDrop<[async fn body@$DIR/large-arg.rs:10:30: 12:2]>`: 3075 bytes, alignment: 1 bytes
 print-type-size     field `.value`: 3075 bytes
 print-type-size type: `std::mem::MaybeUninit<[async fn body@$DIR/large-arg.rs:10:30: 12:2]>`: 3075 bytes, alignment: 1 bytes
@@ -25,14 +25,14 @@ print-type-size         field `.value`: 3075 bytes
 print-type-size type: `[async fn body@$DIR/large-arg.rs:13:26: 15:2]`: 2050 bytes, alignment: 1 bytes
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Suspend0`: 2049 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size         local `.__awaitee`: 1025 bytes
 print-type-size     variant `Returned`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Panicked`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size type: `std::mem::ManuallyDrop<[async fn body@$DIR/large-arg.rs:13:26: 15:2]>`: 2050 bytes, alignment: 1 bytes
 print-type-size     field `.value`: 2050 bytes
 print-type-size type: `std::mem::MaybeUninit<[async fn body@$DIR/large-arg.rs:13:26: 15:2]>`: 2050 bytes, alignment: 1 bytes
@@ -42,11 +42,11 @@ print-type-size         field `.value`: 2050 bytes
 print-type-size type: `[async fn body@$DIR/large-arg.rs:16:26: 18:2]`: 1025 bytes, alignment: 1 bytes
 print-type-size     discriminant: 1 bytes
 print-type-size     variant `Unresumed`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Returned`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size     variant `Panicked`: 1024 bytes
-print-type-size         upvar `.t`: 1024 bytes, offset: 0 bytes, alignment: 1 bytes
+print-type-size         upvar `.t`: 1024 bytes
 print-type-size type: `std::mem::ManuallyDrop<[async fn body@$DIR/large-arg.rs:16:26: 18:2]>`: 1025 bytes, alignment: 1 bytes
 print-type-size     field `.value`: 1025 bytes
 print-type-size type: `std::mem::MaybeUninit<[async fn body@$DIR/large-arg.rs:16:26: 18:2]>`: 1025 bytes, alignment: 1 bytes

diff --git a/tests/ui/consts/const-eval/raw-bytes.32bit.stderr b/tests/ui/consts/const-eval/raw-bytes.32bit.stderr
@@ -465,7 +465,7 @@ LL | const LAYOUT_INVALID_ZERO: Layout = unsafe { Layout::from_size_align_unchec
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
    = note: the raw bytes of the constant (size: 8, align: 4) {
-               00 10 00 00 00 00 00 00                         │ ........
+               00 00 00 00 00 10 00 00                         │ ........
            }
 
 error[E0080]: it is undefined behavior to use this value
@@ -476,7 +476,7 @@ LL | const LAYOUT_INVALID_THREE: Layout = unsafe { Layout::from_size_align_unche
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
    = note: the raw bytes of the constant (size: 8, align: 4) {
-               09 00 00 00 03 00 00 00                         │ ........
+               03 00 00 00 09 00 00 00                         │ ........
            }
 
 error[E0080]: it is undefined behavior to use this value

diff --git a/tests/ui/consts/const-eval/raw-bytes.64bit.stderr b/tests/ui/consts/const-eval/raw-bytes.64bit.stderr
@@ -465,7 +465,7 @@ LL | const LAYOUT_INVALID_ZERO: Layout = unsafe { Layout::from_size_align_unchec
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
    = note: the raw bytes of the constant (size: 16, align: 8) {
-               00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 │ ................
+               00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 │ ................
            }
 
 error[E0080]: it is undefined behavior to use this value
@@ -476,7 +476,7 @@ LL | const LAYOUT_INVALID_THREE: Layout = unsafe { Layout::from_size_align_unche
    |
    = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
    = note: the raw bytes of the constant (size: 16, align: 8) {
-               09 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 │ ................
+               03 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 │ ................
            }
 
 error[E0080]: it is undefined behavior to use this value

diff --git a/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr b/tests/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr
@@ -370,31 +370,31 @@ error: layout_of(NicheFirst) = Layout {
                pref: $PREF_ALIGN,
            },
            abi: ScalarPair(
-               Union {
+               Initialized {
                    value: Int(
                        I8,
                        false,
                    ),
+                   valid_range: 0..=4,
                },
-               Initialized {
+               Union {
                    value: Int(
                        I8,
                        false,
                    ),
-                   valid_range: 0..=4,
                },
            ),
            fields: Arbitrary {
                offsets: [
-                   Size(1 bytes),
+                   Size(0 bytes),
                ],
                memory_index: [
                    0,
                ],
            },
            largest_niche: Some(
                Niche {
-                   offset: Size(1 bytes),
+                   offset: Size(0 bytes),
                    value: Int(
                        I8,
                        false,
@@ -429,29 +429,29 @@ error: layout_of(NicheFirst) = Layout {
                                    I8,
                                    false,
                                ),
-                               valid_range: 0..=255,
+                               valid_range: 0..=2,
                            },
                            Initialized {
                                value: Int(
                                    I8,
                                    false,
                                ),
-                               valid_range: 0..=2,
+                               valid_range: 0..=255,
                            },
                        ),
                        fields: Arbitrary {
                            offsets: [
-                               Size(1 bytes),
                                Size(0 bytes),
+                               Size(1 bytes),
                            ],
                            memory_index: [
-                               1,
                                0,
+                               1,
                            ],
                        },
                        largest_niche: Some(
                            Niche {
-                               offset: Size(1 bytes),
+                               offset: Size(0 bytes),
                                value: Int(
                                    I8,
                                    false,
@@ -514,31 +514,31 @@ error: layout_of(NicheSecond) = Layout {
                pref: $PREF_ALIGN,
            },
            abi: ScalarPair(
-               Union {
+               Initialized {
                    value: Int(
                        I8,
                        false,
                    ),
+                   valid_range: 0..=4,
                },
-               Initialized {
+               Union {
                    value: Int(
                        I8,
                        false,
                    ),
-                   valid_range: 0..=4,
                },
            ),
            fields: Arbitrary {
                offsets: [
-                   Size(1 bytes),
+                   Size(0 bytes),
                ],
                memory_index: [
                    0,
                ],
            },
            largest_niche: Some(
                Niche {
-                   offset: Size(1 bytes),
+                   offset: Size(0 bytes),
                    value: Int(
                        I8,
                        false,
@@ -573,29 +573,29 @@ error: layout_of(NicheSecond) = Layout {
                                    I8,
                                    false,
                                ),
-                               valid_range: 0..=255,
+                               valid_range: 0..=2,
                            },
                            Initialized {
                                value: Int(
                                    I8,
                                    false,
                                ),
-                               valid_range: 0..=2,
+                               valid_range: 0..=255,
                            },
                        ),
                        fields: Arbitrary {
                            offsets: [
-                               Size(0 bytes),
                                Size(1 bytes),
+                               Size(0 bytes),
                            ],
                            memory_index: [
-                               0,
                                1,
+                               0,
                            ],
                        },
                        largest_niche: Some(
                            Niche {
-                               offset: Size(1 bytes),
+                               offset: Size(0 bytes),
                                value: Int(
                                    I8,
                                    false,