diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index b611f9ba8bcb3..9276957dd374c 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -30,7 +30,7 @@ use rustc_middle::ty::{Instance, ParamEnv, Ty, TyCtxt}; use rustc_span::Span; use rustc_span::def_id::DefId; use rustc_target::abi::call::FnAbi; -use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, WasmCAbi}; +use rustc_target::spec::{HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, Target, WasmCAbi}; use crate::common::{SignType, TypeReflection, type_is_pointer}; use crate::context::CodegenCx; @@ -2347,6 +2347,12 @@ impl<'tcx> HasWasmCAbiOpt for Builder<'_, '_, 'tcx> { } } +impl<'tcx> HasS390xVector for Builder<'_, '_, 'tcx> { + fn has_s390x_vector(&self) -> bool { + self.cx.has_s390x_vector() + } +} + pub trait ToGccComp { fn to_gcc_comparison(&self) -> ComparisonOp; } diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 7cb49bf799135..dde121a47ce09 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -19,7 +19,9 @@ use rustc_session::Session; use rustc_span::source_map::respan; use rustc_span::{DUMMY_SP, Span}; use rustc_target::abi::{HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx}; -use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCAbi}; +use rustc_target::spec::{ + HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCAbi, +}; use crate::callee::get_fn; use crate::common::SignType; @@ -538,6 +540,13 @@ impl<'gcc, 'tcx> HasWasmCAbiOpt for CodegenCx<'gcc, 'tcx> { } } +impl<'gcc, 'tcx> HasS390xVector for CodegenCx<'gcc, 'tcx> { + fn has_s390x_vector(&self) -> bool { + // `unstable_target_features` is used here because "vector" is gated behind s390x_target_feature. + self.tcx.sess.unstable_target_features.contains(&sym::vector) + } +} + impl<'gcc, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 6c12b691c26c0..c64e799b40e8d 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -21,7 +21,9 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{FieldIdx, TyAbiInterface, VariantIdx, call}; use rustc_target::spec::abi::Abi as SpecAbi; -use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, PanicStrategy, Target, WasmCAbi}; +use rustc_target::spec::{ + HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, PanicStrategy, Target, WasmCAbi, +}; use tracing::debug; use {rustc_abi as abi, rustc_hir as hir}; @@ -544,6 +546,13 @@ impl<'tcx> HasWasmCAbiOpt for TyCtxt<'tcx> { } } +impl<'tcx> HasS390xVector for TyCtxt<'tcx> { + fn has_s390x_vector(&self) -> bool { + // `unstable_target_features` is used here because "vector" is gated behind s390x_target_feature. + self.sess.unstable_target_features.contains(&sym::vector) + } +} + impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> { #[inline] fn tcx(&self) -> TyCtxt<'tcx> { @@ -595,6 +604,12 @@ impl<'tcx> HasWasmCAbiOpt for LayoutCx<'tcx> { } } +impl<'tcx> HasS390xVector for LayoutCx<'tcx> { + fn has_s390x_vector(&self) -> bool { + self.calc.cx.has_s390x_vector() + } +} + impl<'tcx> HasTyCtxt<'tcx> for LayoutCx<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.calc.cx diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 8e0009695db69..976a63f1991e6 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2126,6 +2126,7 @@ symbols! { vec_pop, vec_with_capacity, vecdeque_iter, + vector, version, vfp2, vis, diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 352861c5ccb49..02615dea47793 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -5,7 +5,7 @@ use rustc_macros::HashStable_Generic; use rustc_span::Symbol; use crate::abi::{self, Abi, Align, FieldsShape, HasDataLayout, Size, TyAbiInterface, TyAndLayout}; -use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi}; +use crate::spec::{self, HasS390xVector, HasTargetSpec, HasWasmCAbiOpt, WasmCAbi}; mod aarch64; mod amdgpu; @@ -876,7 +876,7 @@ impl<'a, Ty> FnAbi<'a, Ty> { ) -> Result<(), AdjustForForeignAbiError> where Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt, + C: HasDataLayout + HasTargetSpec + HasWasmCAbiOpt + HasS390xVector, { if abi == spec::abi::Abi::X86Interrupt { if let Some(arg) = self.args.first_mut() { diff --git a/compiler/rustc_target/src/abi/call/s390x.rs b/compiler/rustc_target/src/abi/call/s390x.rs index 502e733126777..42b18dfcc1624 100644 --- a/compiler/rustc_target/src/abi/call/s390x.rs +++ b/compiler/rustc_target/src/abi/call/s390x.rs @@ -1,19 +1,49 @@ -// FIXME: The assumes we're using the non-vector ABI, i.e., compiling -// for a pre-z13 machine or using -mno-vx. +use crate::abi::call::{ArgAbi, FnAbi, Reg, RegKind}; +use crate::abi::{Abi, HasDataLayout, TyAbiInterface, TyAndLayout}; +use crate::spec::{HasS390xVector, HasTargetSpec}; -use crate::abi::call::{ArgAbi, FnAbi, Reg}; -use crate::abi::{HasDataLayout, TyAbiInterface}; -use crate::spec::HasTargetSpec; +#[derive(Debug, Clone, Copy, PartialEq)] +enum ABI { + NoVector, // no-vector ABI, i.e., compiling for a pre-z13 machine or using -C target-feature=-vector + Vector, // vector ABI, i.e., compiling for a z13 or later machine or using -C target-feature=+vector +} +use ABI::*; + +fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + match layout.abi { + Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) => false, + Abi::Vector { .. } => layout.size.bits() == 128, + Abi::Aggregate { .. } => { + for i in 0..layout.fields.count() { + if contains_vector(cx, layout.field(cx, i)) { + return true; + } + } + false + } + } +} -fn classify_ret(ret: &mut ArgAbi<'_, Ty>) { - if !ret.layout.is_aggregate() && ret.layout.size.bits() <= 64 { +fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, abi: ABI) +where + Ty: TyAbiInterface<'a, C> + Copy, +{ + let size = ret.layout.size; + if !ret.layout.is_aggregate() && size.bits() <= 64 { ret.extend_integer_width_to(64); - } else { - ret.make_indirect(); + return; + } + if abi == Vector && size.bits() == 128 && contains_vector(cx, ret.layout) { + ret.cast_to(Reg { kind: RegKind::Vector, size }); + return; } + ret.make_indirect(); } -fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) +fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, abi: ABI) where Ty: TyAbiInterface<'a, C> + Copy, C: HasDataLayout + HasTargetSpec, @@ -32,19 +62,25 @@ where } return; } - if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 { + + let size = arg.layout.size; + if !arg.layout.is_aggregate() && size.bits() <= 64 { arg.extend_integer_width_to(64); return; } + if abi == Vector && size.bits() == 128 && contains_vector(cx, arg.layout) { + arg.cast_to(Reg { kind: RegKind::Vector, size }); + return; + } if arg.layout.is_single_fp_element(cx) { - match arg.layout.size.bytes() { + match size.bytes() { 4 => arg.cast_to(Reg::f32()), 8 => arg.cast_to(Reg::f64()), _ => arg.make_indirect(), } } else { - match arg.layout.size.bytes() { + match size.bytes() { 1 => arg.cast_to(Reg::i8()), 2 => arg.cast_to(Reg::i16()), 4 => arg.cast_to(Reg::i32()), @@ -57,13 +93,15 @@ where pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) where Ty: TyAbiInterface<'a, C> + Copy, - C: HasDataLayout + HasTargetSpec, + C: HasDataLayout + HasTargetSpec + HasS390xVector, { + let abi = if cx.has_s390x_vector() { Vector } else { NoVector }; + if !fn_abi.ret.is_ignore() { - classify_ret(&mut fn_abi.ret); + classify_ret(cx, &mut fn_abi.ret, abi); } for arg in fn_abi.args.iter_mut() { - classify_arg(cx, arg); + classify_arg(cx, arg, abi); } } diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 82e11a3afce32..6c23e91b2cd4b 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -2096,6 +2096,10 @@ pub trait HasWasmCAbiOpt { fn wasm_c_abi_opt(&self) -> WasmCAbi; } +pub trait HasS390xVector { + fn has_s390x_vector(&self) -> bool; +} + type StaticCow = Cow<'static, T>; /// Optional aspects of a target specification. diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs index 3efbb46483613..a84a18a433ffc 100644 --- a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs +++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs @@ -6,9 +6,6 @@ pub(crate) fn target() -> Target { base.endian = Endian::Big; // z10 is the oldest CPU supported by LLVM base.cpu = "z10".into(); - // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector - // ABI. Pass the -vector feature string to LLVM to respect this assumption. - base.features = "-vector".into(); base.max_atomic_width = Some(128); base.min_global_align = Some(16); base.stack_probes = StackProbeType::Inline; diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs index 65b5c1167bdd8..4bde0fb729c75 100644 --- a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs +++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs @@ -6,9 +6,6 @@ pub(crate) fn target() -> Target { base.endian = Endian::Big; // z10 is the oldest CPU supported by LLVM base.cpu = "z10".into(); - // FIXME: The ABI implementation in abi/call/s390x.rs is for now hard-coded to assume the no-vector - // ABI. Pass the -vector feature string to LLVM to respect this assumption. - base.features = "-vector".into(); base.max_atomic_width = Some(128); base.min_global_align = Some(16); base.static_position_independent_executables = true; diff --git a/tests/assembly/s390x-vector-abi.rs b/tests/assembly/s390x-vector-abi.rs new file mode 100644 index 0000000000000..0503ce8caa68a --- /dev/null +++ b/tests/assembly/s390x-vector-abi.rs @@ -0,0 +1,113 @@ +//@ revisions: z10 z10_vector z13 z13_no_vector +//@ assembly-output: emit-asm +//@ compile-flags: -O +//@[z10] compile-flags: --target s390x-unknown-linux-gnu --cfg no_vector +//@[z10] needs-llvm-components: systemz +//@[z10_vector] compile-flags: --target s390x-unknown-linux-gnu --cfg vector +//@[z10_vector] needs-llvm-components: systemz +//@[z13] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 --cfg vector +//@[z13] needs-llvm-components: systemz +//@[z13_no_vector] compile-flags: --target s390x-unknown-linux-gnu -C target-cpu=z13 -C target-feature=-vector --cfg no_vector +//@[z13_no_vector] needs-llvm-components: systemz + +#![feature(no_core, lang_items, repr_simd)] +#![no_core] +#![crate_type = "lib"] +#![allow(non_camel_case_types)] + +#[lang = "sized"] +trait Sized {} +#[lang = "copy"] +trait Copy {} +#[lang = "freeze"] +trait Freeze {} + +impl Copy for [T; N] {} + +#[repr(simd)] +pub struct i8x8([i8; 8]); +#[repr(simd)] +pub struct i8x16([i8; 16]); +#[repr(simd)] +pub struct i8x32([i8; 32]); +pub struct Wrapper(T); + +impl Copy for i8 {} +impl Copy for i8x8 {} +impl Copy for i8x16 {} +impl Copy for i8x32 {} +impl Copy for Wrapper {} + +// FIXME: ?? +// CHECK-LABEL: vector_small +// no_vector: lg %r0, 0(%r3) +// no_vector-NEXT: stg %r0, 0(%r2) +// no_vector-NEXT: br %r14 +// vector: vlrepg %v24, 0(%r2) +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector_small(x: &i8x8) -> i8x8 { + *x +} +// CHECK-LABEL: vector +// no_vector: mvc 8(8,%r2), 8(%r3) +// no_vector-NEXT: mvc 0(8,%r2), 0(%r3) +// no_vector-NEXT: br %r14 +// vector: vl %v24, 0(%r2), 3 +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector(x: &i8x16) -> i8x16 { + *x +} +// CHECK-LABEL: vector_large +// no_vector: mvc 24(8,%r2), 24(%r3) +// no_vector-NEXT: mvc 16(8,%r2), 16(%r3) +// no_vector-NEXT: mvc 8(8,%r2), 8(%r3) +// no_vector-NEXT: mvc 0(8,%r2), 0(%r3) +// no_vector-NEXT: br %r14 +// vector: vl %v0, 0(%r3), 4 +// vector-NEXT: vl %v1, 16(%r3), 4 +// vector-NEXT: vst %v1, 16(%r2), 4 +// vector-NEXT: vst %v0, 0(%r2), 4 +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector_large(x: &i8x32) -> i8x32 { + *x +} + +// FIXME: ?? +// CHECK-LABEL: vector_wrapper_small +// no_vector: lg %r0, 0(%r3) +// no_vector-NEXT: stg %r0, 0(%r2) +// no_vector-NEXT: br %r14 +// vector: vlrepg %v24, 0(%r2) +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector_wrapper_small(x: &Wrapper) -> Wrapper { + *x +} +// CHECK-LABEL: vector_wrapper +// no_vector: mvc 8(8,%r2), 8(%r3) +// no_vector-NEXT: mvc 0(8,%r2), 0(%r3) +// no_vector-NEXT: br %r14 +// vector: vl %v24, 0(%r2), 3 +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector_wrapper(x: &Wrapper) -> Wrapper { + *x +} +// CHECK-LABEL: vector_wrapper_large +// no_vector: mvc 24(8,%r2), 24(%r3) +// no_vector-NEXT: mvc 16(8,%r2), 16(%r3) +// no_vector-NEXT: mvc 8(8,%r2), 8(%r3) +// no_vector-NEXT: mvc 0(8,%r2), 0(%r3) +// no_vector-NEXT: br %r14 +// vector: vl %v0, 0(%r3), 4 +// vector-NEXT: vl %v1, 16(%r3), 4 +// vector-NEXT: vst %v1, 16(%r2), 4 +// vector-NEXT: vst %v0, 0(%r2), 4 +// vector-NEXT: br %r14 +#[no_mangle] +extern "C" fn vector_wrapper_large(x: &Wrapper) -> Wrapper { + *x +}