Enable inline stack probes on PowerPC and SystemZ

compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-freebsd".into(),

compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-musl".into(),

compiler/rustc_target/src/spec/powerpc64_unknown_openbsd.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-openbsd".into(),

compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/powerpc64le_unknown_freebsd.rs

@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-freebsd".into(),

compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs

@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs

@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-linux-musl".into(),

compiler/rustc_target/src/spec/powerpc_unknown_freebsd.rs

@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     // Extra hint to linker that we are generating secure-PLT code.
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "--target=powerpc-unknown-freebsd13.0"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-freebsd13.0".into(),

compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mspe"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnuspe".into(),

compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-musl".into(),

compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-netbsd".into(),

compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.endian = Endian::Big;
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-openbsd".into(),

compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "--secure-plt"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs

@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mspe", "--secure-plt"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnuspe".into(),

compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs

@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
@@ -12,6 +12,7 @@ pub fn target() -> Target {
     base.features = "-vector".into();
     base.max_atomic_width = Some(64);
     base.min_global_align = Some(16);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "s390x-unknown-linux-gnu".into(),

compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs

@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
@@ -13,6 +13,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.min_global_align = Some(16);
     base.static_position_independent_executables = true;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "s390x-unknown-linux-musl".into(),

src/test/codegen/stack-probes-call.rs

@@ -0,0 +1,22 @@
+// Check the "probe-stack" attribute for targets with `StackProbeType::Call`,
+// or `StackProbeType::InlineOrCall` when running on older LLVM.
+
+// compile-flags: -C no-prepopulate-passes
+// revisions: i686 x86_64
+//[i686] compile-flags: --target i686-unknown-linux-gnu
+//[i686] needs-llvm-components: x86
+//[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//[x86_64] needs-llvm-components: x86
+
+#![crate_type = "rlib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="__rust_probestack"{{.*}} }
+}

src/test/codegen/stack-probes-inline.rs

@@ -0,0 +1,26 @@
+// Check the "probe-stack" attribute for targets with `StackProbeType::Inline`,
+// or `StackProbeType::InlineOrCall` when running on newer LLVM.
+
+// compile-flags: -C no-prepopulate-passes
+// revisions: powerpc powerpc64 powerpc64le s390x
+//[powerpc] compile-flags: --target powerpc-unknown-linux-gnu
+//[powerpc] needs-llvm-components: powerpc
+//[powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//[powerpc64] needs-llvm-components: powerpc
+//[powerpc64le] compile-flags: --target powerpc64le-unknown-linux-gnu
+//[powerpc64le] needs-llvm-components: powerpc
+//[s390x] compile-flags: --target s390x-unknown-linux-gnu
+//[s390x] needs-llvm-components: systemz
+
+#![crate_type = "rlib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="inline-asm"{{.*}} }
+}

src/test/ui/abi/stack-probes-lto.rs

@@ -3,8 +3,6 @@
 // ignore-aarch64
 // ignore-mips
 // ignore-mips64
-// ignore-powerpc
-// ignore-s390x
 // ignore-sparc
 // ignore-sparc64
 // ignore-wasm

src/test/ui/abi/stack-probes.rs

@@ -3,8 +3,6 @@
 // ignore-aarch64
 // ignore-mips
 // ignore-mips64
-// ignore-powerpc
-// ignore-s390x
 // ignore-sparc
 // ignore-sparc64
 // ignore-wasm
@@ -27,8 +25,9 @@ fn main() {
     let args = env::args().skip(1).collect::<Vec<_>>();
     if args.len() > 0 {
         match &args[0][..] {
-            "main-thread" => recurse(&MaybeUninit::uninit()),
-            "child-thread" => thread::spawn(|| recurse(&MaybeUninit::uninit())).join().unwrap(),
+            "main-recurse" => overflow_recurse(),
+            "child-recurse" => thread::spawn(overflow_recurse).join().unwrap(),
+            "child-frame" => overflow_frame(),
             _ => panic!(),
         }
         return;
@@ -41,9 +40,10 @@ fn main() {
     // that we report stack overflow on the main thread, see #43052 for some
     // details
     if cfg!(not(target_os = "linux")) {
-        assert_overflow(Command::new(&me).arg("main-thread"));
+        assert_overflow(Command::new(&me).arg("main-recurse"));
     }
-    assert_overflow(Command::new(&me).arg("child-thread"));
+    assert_overflow(Command::new(&me).arg("child-recurse"));
+    assert_overflow(Command::new(&me).arg("child-frame"));
 }
 
 #[allow(unconditional_recursion)]
@@ -55,6 +55,23 @@ fn recurse(array: &MaybeUninit<[u64; 1024]>) {
     recurse(&local);
 }
 
+#[inline(never)]
+fn overflow_recurse() {
+    recurse(&MaybeUninit::uninit());
+}
+
+fn overflow_frame() {
+    // By using a 1MiB stack frame with only 512KiB stack, we'll jump over any
+    // guard page, even with 64K pages -- but stack probes should catch it.
+    const STACK_SIZE: usize = 512 * 1024;
+    thread::Builder::new().stack_size(STACK_SIZE).spawn(|| {
+        let local: MaybeUninit<[u8; 2 * STACK_SIZE]> = MaybeUninit::uninit();
+        unsafe {
+            black_box(local.as_ptr() as u64);
+        }
+    }).unwrap().join().unwrap();
+}
+
 fn assert_overflow(cmd: &mut Command) {
     let output = cmd.output().unwrap();
     assert!(!output.status.success());