From 50fd17ba705301ebc593767b44de5ba276650122 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Tue, 16 Jul 2024 20:14:51 -0400 Subject: [PATCH 1/2] Add MultiplyAddRotateComplexBySelectedScalar --- src/coreclr/jit/codegenarm64test.cpp | 6 +- src/coreclr/jit/emitarm64sve.cpp | 13 +- src/coreclr/jit/hwintrinsic.cpp | 4 +- src/coreclr/jit/hwintrinsic.h | 13 + src/coreclr/jit/hwintrinsicarm64.cpp | 66 ++ src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 60 ++ src/coreclr/jit/hwintrinsiclistarm64sve.h | 1 + src/coreclr/jit/lowerarmarch.cpp | 12 + src/coreclr/jit/lsraarm64.cpp | 39 +- .../Arm/Sve.PlatformNotSupported.cs | 8 + .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 8 + .../ref/System.Runtime.Intrinsics.cs | 2 + .../GenerateHWIntrinsicTests_Arm.cs | 10 + .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 27 +- .../_SveImm2TernOpTestTemplate.template | 623 ++++++++++++++++++ 15 files changed, 864 insertions(+), 28 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index e136de5a07595..3d09f1023ff5f 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -6592,11 +6592,11 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_GV_3A theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V0, REG_V1, REG_V0, 0, 0, INS_OPTS_SCALABLE_S); // FCMLA .S, .S, .S[], - theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V2, REG_V3, REG_V5, 1, 90, + theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V2, REG_V3, REG_V5, 1, 1, INS_OPTS_SCALABLE_S); // FCMLA .S, .S, .S[], - theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V4, REG_V5, REG_V10, 0, 180, + theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V4, REG_V5, REG_V10, 0, 2, INS_OPTS_SCALABLE_S); // FCMLA .S, .S, .S[], - theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V6, REG_V7, REG_V15, 1, 270, + theEmitter->emitIns_R_R_R_I_I(INS_sve_fcmla, EA_SCALABLE, REG_V6, REG_V7, REG_V15, 1, 3, INS_OPTS_SCALABLE_S); // FCMLA .S, .S, .S[], // IF_SVE_GX_3A diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 6fb6723eb6381..714c5716e1cd1 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -5825,14 +5825,13 @@ void emitter::emitInsSve_R_R_R_I_I(instruction ins, case INS_sve_fcmla: assert(opt == INS_OPTS_SCALABLE_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidUimm<1>(imm1)); // i - assert(isValidRot(imm2)); // rr + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + assert(isValidUimm<1>(imm1)); // i + assert(emitIsValidEncodedRotationImm0_to_270(imm2)); // rr - // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) - imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + imm = (imm1 << 2) | imm2; fmt = IF_SVE_GV_3A; break; diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 4d31cebad96b3..fcec214145893 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -963,9 +963,7 @@ static void ValidateHWIntrinsicInfo(CORINFO_InstructionSet isa, NamedIntrinsic n if (info.numArgs != -1) { // We should only have an expected number of arguments -#if defined(TARGET_ARM64) - assert((info.numArgs >= 0) && (info.numArgs <= 4)); -#elif defined(TARGET_XARCH) +#if defined(TARGET_ARM64) || defined(TARGET_XARCH) assert((info.numArgs >= 0) && (info.numArgs <= 5)); #else unreached(); diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 52cb9eb9a4806..2491fbdcd4115 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1085,6 +1085,14 @@ struct HWIntrinsicInfo break; } + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + { + assert(sig->numArgs == 5); + *imm1Pos = 0; + *imm2Pos = 1; + break; + } + default: { assert(sig->numArgs > 0); @@ -1105,6 +1113,7 @@ struct HWIntrinsic final , op2(nullptr) , op3(nullptr) , op4(nullptr) + , op5(nullptr) , numOperands(0) , baseType(TYP_UNDEF) { @@ -1134,6 +1143,7 @@ struct HWIntrinsic final GenTree* op2; GenTree* op3; GenTree* op4; + GenTree* op5; size_t numOperands; var_types baseType; @@ -1144,6 +1154,9 @@ struct HWIntrinsic final switch (numOperands) { + case 5: + op5 = node->Op(5); + FALLTHROUGH; case 4: op4 = node->Op(4); FALLTHROUGH; diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index abe215d80a552..384c2fc4f3b1d 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -514,6 +514,27 @@ void HWIntrinsicInfo::lookupImmBounds( immUpperBound = 3; break; + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + // rotation comes after index in the intrinsic's signature, + // but flip the order here so we check the larger range first. + // This conforms to the existing logic in LinearScan::BuildHWIntrinsic + // when determining if we need an internal register for the jump table. + // This flipped ordering is reflected in HWIntrinsicInfo::GetImmOpsPositions. + if (immNumber == 1) + { + // Bounds for rotation + immLowerBound = 0; + immUpperBound = 3; + } + else + { + // Bounds for index + assert(immNumber == 2); + immLowerBound = 0; + immUpperBound = 1; + } + break; + case NI_Sve_TrigonometricMultiplyAddCoefficient: immLowerBound = 0; immUpperBound = 7; @@ -3004,6 +3025,51 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + { + assert(sig->numArgs == 5); + assert(!isScalar); + + CORINFO_ARG_LIST_HANDLE arg1 = sig->args; + CORINFO_ARG_LIST_HANDLE arg2 = info.compCompHnd->getArgNext(arg1); + CORINFO_ARG_LIST_HANDLE arg3 = info.compCompHnd->getArgNext(arg2); + CORINFO_ARG_LIST_HANDLE arg4 = info.compCompHnd->getArgNext(arg3); + CORINFO_ARG_LIST_HANDLE arg5 = info.compCompHnd->getArgNext(arg4); + var_types argType = TYP_UNKNOWN; + CORINFO_CLASS_HANDLE argClass = NO_CLASS_HANDLE; + + int imm1LowerBound, imm1UpperBound; // Range for rotation + int imm2LowerBound, imm2UpperBound; // Range for index + HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, simdBaseType, 1, &imm1LowerBound, &imm1UpperBound); + HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, simdBaseType, 2, &imm2LowerBound, &imm2UpperBound); + + argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg5, &argClass))); + GenTree* op5 = getArgForHWIntrinsic(argType, argClass); + assert(HWIntrinsicInfo::isImmOp(intrinsic, op5)); + op5 = addRangeCheckIfNeeded(intrinsic, op5, mustExpand, imm1LowerBound, imm1UpperBound); + + argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg4, &argClass))); + op4 = getArgForHWIntrinsic(argType, argClass); + assert(HWIntrinsicInfo::isImmOp(intrinsic, op4)); + op4 = addRangeCheckIfNeeded(intrinsic, op4, mustExpand, imm2LowerBound, imm2UpperBound); + + argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass))); + op3 = getArgForHWIntrinsic(argType, argClass); + argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg2, &argClass))); + op2 = getArgForHWIntrinsic(argType, argClass); + argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass))); + op1 = getArgForHWIntrinsic(argType, argClass); + + SetOpLclRelatedToSIMDIntrinsic(op1); + SetOpLclRelatedToSIMDIntrinsic(op2); + SetOpLclRelatedToSIMDIntrinsic(op3); + SetOpLclRelatedToSIMDIntrinsic(op4); + SetOpLclRelatedToSIMDIntrinsic(op5); + retNode = new (this, GT_HWINTRINSIC) GenTreeHWIntrinsic(retType, getAllocator(CMK_ASTNode), intrinsic, + simdBaseJitType, simdSize, op1, op2, op3, op4, op5); + break; + } + default: { return nullptr; diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index f8e77197916df..6833008175f3f 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -265,9 +265,15 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) regNumber op2Reg = REG_NA; regNumber op3Reg = REG_NA; regNumber op4Reg = REG_NA; + regNumber op5Reg = REG_NA; switch (intrin.numOperands) { + case 5: + assert(intrin.op5 != nullptr); + op5Reg = intrin.op5->GetRegNum(); + FALLTHROUGH; + case 4: assert(intrin.op4 != nullptr); op4Reg = intrin.op4->GetRegNum(); @@ -2407,6 +2413,60 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + { + assert(isRMW); + assert(hasImmediateOperand); + + if (targetReg != op1Reg) + { + assert(targetReg != op2Reg); + assert(targetReg != op3Reg); + GetEmitter()->emitInsSve_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, op1Reg); + } + + // If both immediates are constant, we don't need a jump table + if (intrin.op4->IsCnsIntOrI() && intrin.op5->IsCnsIntOrI()) + { + assert(intrin.op4->isContainedIntOrIImmed() && intrin.op5->isContainedIntOrIImmed()); + GetEmitter()->emitInsSve_R_R_R_I_I(ins, emitSize, targetReg, op2Reg, op3Reg, + intrin.op4->AsIntCon()->gtIconVal, + intrin.op5->AsIntCon()->gtIconVal, opt); + } + else + { + // Use the helper to generate a table. The table can only use a single lookup value, therefore + // the two immediates index (0 to 1, in op4Reg) and rotation (0 to 3, in op5Reg) must be + // combined to a single value (0 to 7) + assert(!intrin.op4->isContainedIntOrIImmed() && !intrin.op5->isContainedIntOrIImmed()); + emitAttr scalarSize = emitActualTypeSize(node->GetSimdBaseType()); + + // Combine the two immediates into op4Reg + // Shift rotation left to be out of range of index + GetEmitter()->emitIns_R_R_I(INS_lsl, scalarSize, op5Reg, op5Reg, 1); + // Combine the two values by ORing + GetEmitter()->emitIns_R_R_R(INS_orr, scalarSize, op4Reg, op4Reg, op5Reg); + + // Generate the table using the combined immediate + HWIntrinsicImmOpHelper helper(this, op4Reg, 0, 7, node); + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + // Extract index and rotation from the immediate + const int value = helper.ImmValue(); + const ssize_t index = value & 1; + const ssize_t rotation = value >> 1; + GetEmitter()->emitInsSve_R_R_R_I_I(ins, emitSize, targetReg, op2Reg, op3Reg, index, rotation, + opt); + } + + // Restore the original values in op4Reg and op5Reg + GetEmitter()->emitIns_R_R_I(INS_and, scalarSize, op4Reg, op4Reg, 1); + GetEmitter()->emitIns_R_R_I(INS_lsr, scalarSize, op5Reg, op5Reg, 1); + } + + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 961076326818a..5590ef282815e 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -190,6 +190,7 @@ HARDWARE_INTRINSIC(Sve, MinNumberAcross, HARDWARE_INTRINSIC(Sve, Multiply, -1, 2, true, {INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_mul, INS_sve_fmul, INS_sve_fmul}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, MultiplyAdd, -1, -1, false, {INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_sve_mla, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplex, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_sve_fcmla}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(Sve, MultiplyAddRotateComplexBySelectedScalar, -1, 5, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fcmla, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(Sve, MultiplyBySelectedScalar, -1, 3, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmul, INS_sve_fmul}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation) HARDWARE_INTRINSIC(Sve, MultiplyExtended, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_fmulx, INS_sve_fmulx}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, MultiplySubtract, -1, -1, false, {INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen) diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index e64b134192ae2..cd5032921804d 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -3698,6 +3698,18 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } break; + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + assert(hasImmediateOperand); + assert(varTypeIsIntegral(intrin.op4)); + assert(varTypeIsIntegral(intrin.op5)); + // Can only avoid generating a table if both immediates are constant. + if (intrin.op4->IsCnsIntOrI() && intrin.op5->IsCnsIntOrI()) + { + MakeSrcContained(node, intrin.op4); + MakeSrcContained(node, intrin.op5); + } + break; + default: unreached(); } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 266cb47537d50..a4ed3ac6cac53 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1530,7 +1530,15 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou // Can only avoid generating a table if both immediates are constant. assert(intrin.op2->isContainedIntOrIImmed() == intrin.op3->isContainedIntOrIImmed()); needBranchTargetReg = !intrin.op2->isContainedIntOrIImmed(); - // Ensure that internal does not collide with desination. + // Ensure that internal does not collide with destination. + setInternalRegsDelayFree = true; + break; + + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + // Can only avoid generating a table if both immediates are constant. + assert(intrin.op4->isContainedIntOrIImmed() == intrin.op5->isContainedIntOrIImmed()); + needBranchTargetReg = !intrin.op4->isContainedIntOrIImmed(); + // Ensure that internal does not collide with destination. setInternalRegsDelayFree = true; break; @@ -1949,40 +1957,38 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou // Special handling for embedded intrinsics with immediates: // We might need an additional register to hold branch targets into the switch table // that encodes the immediate + bool needsInternalRegister; switch (intrinEmb.id) { case NI_Sve_ShiftRightArithmeticForDivide: assert(embHasImmediateOperand); assert(numArgs == 2); - if (!embOp2Node->Op(2)->isContainedIntOrIImmed()) - { - buildInternalIntRegisterDefForNode(embOp2Node); - } + needsInternalRegister = !embOp2Node->Op(2)->isContainedIntOrIImmed(); break; case NI_Sve_AddRotateComplex: assert(embHasImmediateOperand); assert(numArgs == 3); - if (!embOp2Node->Op(3)->isContainedIntOrIImmed()) - { - buildInternalIntRegisterDefForNode(embOp2Node); - } + needsInternalRegister = !embOp2Node->Op(3)->isContainedIntOrIImmed(); break; case NI_Sve_MultiplyAddRotateComplex: assert(embHasImmediateOperand); assert(numArgs == 4); - if (!embOp2Node->Op(4)->isContainedIntOrIImmed()) - { - buildInternalIntRegisterDefForNode(embOp2Node); - } + needsInternalRegister = !embOp2Node->Op(4)->isContainedIntOrIImmed(); break; default: assert(!embHasImmediateOperand); + needsInternalRegister = false; break; } + if (needsInternalRegister) + { + buildInternalIntRegisterDefForNode(embOp2Node); + } + tgtPrefUse = BuildUse(embOp2Node->Op(1)); srcCount += 1; @@ -2105,6 +2111,12 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou assert(lowVectorOperandNum != 4); assert(!tgtPrefOp2); srcCount += isRMW ? BuildDelayFreeUses(intrin.op4, intrin.op1) : BuildOperandUses(intrin.op4); + + if (intrin.op5 != nullptr) + { + assert(isRMW); + srcCount += BuildDelayFreeUses(intrin.op5, intrin.op1); + } } } } @@ -2426,6 +2438,7 @@ void LinearScan::getLowVectorOperandAndCandidates(HWIntrinsic intrin, size_t* op case NI_Sve_DotProductBySelectedScalar: case NI_Sve_FusedMultiplyAddBySelectedScalar: case NI_Sve_FusedMultiplySubtractBySelectedScalar: + case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: *operandNum = 3; break; case NI_Sve_MultiplyBySelectedScalar: diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 3d52204ba67f2..c559123fc2be8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -5854,6 +5854,14 @@ internal Arm64() { } /// public static unsafe Vector MultiplyAddRotateComplex(Vector addend, Vector left, Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw new PlatformNotSupportedException(); } + /// Complex multiply-add with rotate + + /// + /// svfloat32_t svcmla_lane[_f32](svfloat32_t op1, svfloat32_t op2, svfloat32_t op3, uint64_t imm_index, uint64_t imm_rotation) + /// FCMLA Ztied1.S, Zop2.S, Zop3.S[imm_index], #imm_rotation + /// + public static unsafe Vector MultiplyAddRotateComplexBySelectedScalar(Vector addend, Vector left, Vector right, [ConstantExpected(Min = 0, Max = (byte)(1))] byte rightIndex, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw new PlatformNotSupportedException(); } + /// MultiplyBySelectedScalar : Multiply /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index e7dc14f3a3f2c..cb5c1a4fc21c8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -5909,6 +5909,14 @@ internal Arm64() { } /// public static unsafe Vector MultiplyAddRotateComplex(Vector addend, Vector left, Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) => MultiplyAddRotateComplex(addend, left, right, rotation); + /// Complex multiply-add with rotate + + /// + /// svfloat32_t svcmla_lane[_f32](svfloat32_t op1, svfloat32_t op2, svfloat32_t op3, uint64_t imm_index, uint64_t imm_rotation) + /// FCMLA Ztied1.S, Zop2.S, Zop3.S[imm_index], #imm_rotation + /// + public static unsafe Vector MultiplyAddRotateComplexBySelectedScalar(Vector addend, Vector left, Vector right, [ConstantExpected(Min = 0, Max = (byte)(1))] byte rightIndex, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) => MultiplyAddRotateComplexBySelectedScalar(addend, left, right, rightIndex, rotation); + /// MultiplyBySelectedScalar : Multiply /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index fc632d99b3387..bcdc8913d2067 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -5194,6 +5194,8 @@ internal Arm64() { } public static System.Numerics.Vector MultiplyAddRotateComplex(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } public static System.Numerics.Vector MultiplyAddRotateComplex(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } + public static System.Numerics.Vector MultiplyAddRotateComplexBySelectedScalar(System.Numerics.Vector addend, System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected(Min = 0, Max = (byte)(1))] byte rightIndex, [ConstantExpected(Min = 0, Max = (byte)(3))] byte rotation) { throw null; } + public static System.Numerics.Vector MultiplyBySelectedScalar(System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected] byte rightIndex) { throw null; } public static System.Numerics.Vector MultiplyBySelectedScalar(System.Numerics.Vector left, System.Numerics.Vector right, [ConstantExpected] byte rightIndex) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index c20d6c8dd1bf5..1342855df126e 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -288,6 +288,7 @@ ("_SveTernOpFirstArgTestTemplate.template", "SveVecTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }), ("_SveImmTernOpTestTemplate.template", "SveVecImmTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }), ("_SveImmTernOpTestTemplate.template", "SveVecImmTernOpVecTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_VectorValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleVecOpTest_VectorValidationLogicForCndSel_FalseValue }), + ("_SveImm2TernOpTestTemplate.template", "SveVecImm2TernOpVecTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_VectorValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleVecOpTest_VectorValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleVecOpTest_VectorValidationLogicForCndSel_FalseValue }), ("_SveTernOpMaskedOpTestTemplate.template", "SveVecTernOpMaskedTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }), ("_SveImmTernOpFirstArgTestTemplate.template", "SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel, ["TemplateValidationLogicForCndSel_FalseValue"] = SimpleTernVecOpTest_ValidationLogicForCndSel_FalseValue }), ("_SveScalarTernOpTestTemplate.template", "SveScalarTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleScalarOpTest_ValidationLogic }), @@ -3877,6 +3878,15 @@ ("SveVecImmTernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplex_double_2", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplex", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Double",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetDouble()",["NextValueOp2"] = "TestLibrary.Generator.GetDouble()",["NextValueOp3"] = "TestLibrary.Generator.GetDouble()", ["NextValueMask"] = "Helpers.getMaskDouble()", ["Imm"] = "2", ["InvalidImm"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplex(firstOp, secondOp, thirdOp, Imm))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplex(first, second, third, Imm)"}), ("SveVecImmTernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplex_double_3", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplex", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Double",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetDouble()",["NextValueOp2"] = "TestLibrary.Generator.GetDouble()",["NextValueOp3"] = "TestLibrary.Generator.GetDouble()", ["NextValueMask"] = "Helpers.getMaskDouble()", ["Imm"] = "3", ["InvalidImm"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplex(firstOp, secondOp, thirdOp, Imm))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplex(first, second, third, Imm)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_0_0", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "0", ["Imm2"] = "0", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_0_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "0", ["Imm2"] = "1", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_0_2", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "0", ["Imm2"] = "2", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_0_3", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "0", ["Imm2"] = "3", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_1_0", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "1", ["Imm2"] = "0", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_1_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "1", ["Imm2"] = "1", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_1_2", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "1", ["Imm2"] = "2", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImm2TernOpVecTest.template", new Dictionary {["TestName"] = "Sve_MultiplyAddRotateComplexBySelectedScalar_float_1_3", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyAddRotateComplexBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single",["LargestVectorSize"] = "64",["NextValueOp1"] = "TestLibrary.Generator.GetSingle()",["NextValueOp2"] = "TestLibrary.Generator.GetSingle()",["NextValueOp3"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm1"] = "1", ["Imm2"] = "3", ["InvalidImm1"] = "2", ["InvalidImm2"] = "4", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.MultiplyAddRotateComplexBySelectedScalar(firstOp, secondOp, thirdOp, Imm1, Imm2))", ["GetVectorResult"] = "Helpers.MultiplyAddRotateComplexBySelectedScalar(first, second, third, Imm1, Imm2)"}), + ("SveVecImmBinOpTest.template", new Dictionary {["TestName"] = "Sve_MultiplyBySelectedScalar_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["NextValueMask"] = "Helpers.getMaskSingle()", ["Imm"] = "1", ["InvalidImm"] = "4", ["ValidateIterResult"] = "BitConverter.SingleToInt32Bits(Helpers.Multiply(firstOp[i], secondOp[Imm])) != BitConverter.SingleToInt32Bits(result[i])",["GetIterResult"] = "Helpers.Multiply(firstOp[i], secondOp[Imm])", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), ("SveVecImmBinOpTest.template", new Dictionary {["TestName"] = "Sve_MultiplyBySelectedScalar_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "MultiplyBySelectedScalar", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["NextValueMask"] = "Helpers.getMaskDouble()", ["Imm"] = "0", ["InvalidImm"] = "2", ["ValidateIterResult"] = "BitConverter.DoubleToInt64Bits(Helpers.Multiply(firstOp[i], secondOp[Imm])) != BitConverter.DoubleToInt64Bits(result[i])",["GetIterResult"] = "Helpers.Multiply(firstOp[i], secondOp[Imm])", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 1e3b4c342b1a3..7e43078c24316 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -5243,7 +5243,7 @@ public static float[] MultiplyAddRotateComplex(float[] op1, float[] op2, float[] (float ans1, float ans2) = imm switch { 0 => (FusedMultiplyAdd(op1[real], op2[real], op3[real]), FusedMultiplyAdd(op1[img], op2[real], op3[img])), - 1 => (FusedMultiplySubtract(op1[real], op2[img], op3[img]), FusedMultiplyAdd(op1[img], op2[img], op3[i])), + 1 => (FusedMultiplySubtract(op1[real], op2[img], op3[img]), FusedMultiplyAdd(op1[img], op2[img], op3[real])), 2 => (FusedMultiplySubtract(op1[real], op2[real], op3[real]), FusedMultiplySubtract(op1[img], op2[real], op3[img])), 3 => (FusedMultiplyAdd(op1[real], op2[img], op3[img]), FusedMultiplySubtract(op1[img], op2[img], op3[real])), _ => (0.0f, 0.0f) @@ -5256,6 +5256,29 @@ public static float[] MultiplyAddRotateComplex(float[] op1, float[] op2, float[] return op1; } + public static float[] MultiplyAddRotateComplexBySelectedScalar(float[] op1, float[] op2, float[] op3, byte index, byte imm) + { + for (int i = 0; i < op1.Length; i += 2) + { + int real = i; + int img = i + 1; + (float op3Real, float op3Img) = (op3[index * 2], op3[(index * 2) + 1]); + (float ans1, float ans2) = imm switch + { + 0 => (FusedMultiplyAdd(op1[real], op2[real], op3Real), FusedMultiplyAdd(op1[img], op2[real], op3Img)), + 1 => (FusedMultiplySubtract(op1[real], op2[img], op3Img), FusedMultiplyAdd(op1[img], op2[img], op3Real)), + 2 => (FusedMultiplySubtract(op1[real], op2[real], op3Real), FusedMultiplySubtract(op1[img], op2[real], op3Img)), + 3 => (FusedMultiplyAdd(op1[real], op2[img], op3Img), FusedMultiplySubtract(op1[img], op2[img], op3Real)), + _ => (0.0f, 0.0f) + }; + + op1[real] = ans1; + op1[img] = ans2; + } + + return op1; + } + public static float MultiplyExtended(float op1, float op2) { bool inf1 = float.IsInfinity(op1); @@ -5442,7 +5465,7 @@ public static double[] MultiplyAddRotateComplex(double[] op1, double[] op2, doub (double ans1, double ans2) = imm switch { 0 => (FusedMultiplyAdd(op1[real], op2[real], op3[real]), FusedMultiplyAdd(op1[img], op2[real], op3[img])), - 1 => (FusedMultiplySubtract(op1[real], op2[img], op3[img]), FusedMultiplyAdd(op1[img], op2[img], op3[i])), + 1 => (FusedMultiplySubtract(op1[real], op2[img], op3[img]), FusedMultiplyAdd(op1[img], op2[img], op3[real])), 2 => (FusedMultiplySubtract(op1[real], op2[real], op3[real]), FusedMultiplySubtract(op1[img], op2[real], op3[img])), 3 => (FusedMultiplyAdd(op1[real], op2[img], op3[img]), FusedMultiplySubtract(op1[img], op2[img], op3[real])), _ => (0.0, 0.0) diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template new file mode 100644 index 0000000000000..aaa8c0718bee0 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template @@ -0,0 +1,623 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics.Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {TemplateName}TernaryOpTest__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({LoadIsa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op2 as falseValue + test.ConditionalSelect_Op2(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_Op3(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + + // Validates basic functionality fails with invalid immediates, using Unsafe.ReadUnaligned + test.RunBasicScenario_UnsafeRead_InvalidImm1(); + test.RunBasicScenario_UnsafeRead_InvalidImm2(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {TemplateName}TernaryOpTest__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] inArray3; + private byte[] outArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle inHandle3; + private GCHandle outHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op1BaseType}[] inArray2, {Op1BaseType}[] inArray3, {RetBaseType}[] outArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray3 = inArray3.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfinArray3 || (alignment * 2) < sizeOfoutArray) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.inArray3 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.inHandle3 = GCHandle.Alloc(this.inArray3, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray3Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray3[0]), (uint)sizeOfinArray3); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray3Ptr => Align((byte*)(inHandle3.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + inHandle3.Free(); + outHandle.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private struct TestStruct + { + public {Op1VectorType}<{Op1BaseType}> _fld1; + public {Op1VectorType}<{Op1BaseType}> _fld2; + public {Op1VectorType}<{Op1BaseType}> _fld3; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld2), ref Unsafe.As<{Op1BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data3[i] = {NextValueOp3}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref testStruct._fld3), ref Unsafe.As<{Op1BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + return testStruct; + } + + public void RunStructFldScenario({TemplateName}TernaryOpTest__{TestName} testClass) + { + var result = {Isa}.{Method}(_fld1, _fld2, _fld3, {Imm1}, {Imm2}); + + Unsafe.Write(testClass._dataTable.outArrayPtr, result); + testClass.ValidateResult(_fld1, _fld2, _fld3, testClass._dataTable.outArrayPtr); + } + } + + private static readonly int LargestVectorSize = {LargestVectorSize}; + + private static readonly int Op1ElementCount = Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>() / sizeof({Op1BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); + private static readonly byte Imm1 = {Imm1}; + private static readonly byte Imm2 = {Imm2}; + + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data2 = new {Op1BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _data3 = new {Op1BaseType}[Op1ElementCount]; + + private {Op1VectorType}<{Op1BaseType}> _mask; + private {Op1VectorType}<{Op1BaseType}> _fld1; + private {Op1VectorType}<{Op1BaseType}> _fld2; + private {Op1VectorType}<{Op1BaseType}> _fld3; + private {Op1VectorType}<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public {TemplateName}TernaryOpTest__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _maskData[i] = ({Op1BaseType})({NextValueMask} % 2); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld2), ref Unsafe.As<{Op1BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + for (var i = 0; i < Op1ElementCount; i++) { _data3[i] = {NextValueOp3}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _fld3), ref Unsafe.As<{Op1BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1VectorType}<{Op1BaseType}>, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data3[0]), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op1ElementCount; i++) { _data2[i] = {NextValueOp2}; } + for (var i = 0; i < Op1ElementCount; i++) { _data3[i] = {NextValueOp3}; } + _dataTable = new DataTable(_data1, _data2, _data3, new {RetBaseType}[RetElementCount], LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray3Ptr), + {Imm1}, + {Imm2} + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunBasicScenario_UnsafeRead_InvalidImm1() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead_InvalidImm1)); + + bool succeeded = false; + try + { + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray3Ptr), + {InvalidImm1}, + {Imm2} + ); + Console.WriteLine(result); + } + catch (ArgumentOutOfRangeException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + public void RunBasicScenario_UnsafeRead_InvalidImm2() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead_InvalidImm2)); + + bool succeeded = false; + try + { + var result = {Isa}.{Method}( + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray3Ptr), + {Imm1}, + {InvalidImm2} + ); + Console.WriteLine(result); + } + catch (ArgumentOutOfRangeException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + + {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}( + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray2Ptr)), + {LoadIsa}.Load{Op1VectorType}(loadMask, ({Op1BaseType}*)(_dataTable.inArray3Ptr)), + {Imm1}, + {Imm2} + ); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op1VectorType}<{Op1BaseType}>), typeof({Op1VectorType}<{Op1BaseType}>), typeof(byte), typeof(byte) }) + .Invoke(null, new object[] { + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray2Ptr), + Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray3Ptr), + (byte){Imm1}, + (byte){Imm2} + }); + + Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.inArray3Ptr, _dataTable.outArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + var op1 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray1Ptr); + var op2 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray2Ptr); + var op3 = Unsafe.Read<{Op1VectorType}<{Op1BaseType}>>(_dataTable.inArray3Ptr); + var result = {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2}); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(op1, op2, op3, _dataTable.outArrayPtr); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + var result = {Isa}.{Method}(_fld1, _fld2, _fld3, {Imm1}, {Imm2}); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(_fld1, _fld2, _fld3, _dataTable.outArrayPtr); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._fld1, test._fld2, test._fld3, {Imm1}, {Imm2}); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateResult(test._fld1, test._fld2, test._fld3, _dataTable.outArrayPtr); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld1); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld1); + } + + public void ConditionalSelect_Op2() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op2_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld2); + } + + public void ConditionalSelect_Op3() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _fld3); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld3); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld3); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _fld3); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _fld3); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op3_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _fld3); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue"); + ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue"); + ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue"); + ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue"); + ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2}), falseOp); + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> op3, {Op1VectorType}<{Op1BaseType}> trueOp) + { + var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2})); + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult_FalseValue(mask, op1, op2, op3, trueOp, _dataTable.outArrayPtr); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_TrueValue({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> firstOp, {Op1VectorType}<{Op1BaseType}> secondOp, {Op1VectorType}<{Op1BaseType}> thirdOp, {Op1VectorType}<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] first = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] second = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] third = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref first[0]), firstOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref second[0]), secondOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref third[0]), thirdOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + bool succeeded = true; + {TemplateValidationLogicForCndSel} + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" first: ({string.Join(", ", first)})"); + TestLibrary.TestFramework.LogInformation($" second: ({string.Join(", ", second)})"); + TestLibrary.TestFramework.LogInformation($" third: ({string.Join(", ", third)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult_FalseValue({Op1VectorType}<{Op1BaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> firstOp, {Op1VectorType}<{Op1BaseType}> secondOp, {Op1VectorType}<{Op1BaseType}> thirdOp, {Op1VectorType}<{Op1BaseType}> trueOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] first = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] second = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] third = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] trueVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref first[0]), firstOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref second[0]), secondOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref third[0]), thirdOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref trueVal[0]), trueOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + bool succeeded = true; + + {TemplateValidationLogicForCndSel_FalseValue} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" first: ({string.Join(", ", first)})"); + TestLibrary.TestFramework.LogInformation($" second: ({string.Join(", ", second)})"); + TestLibrary.TestFramework.LogInformation($" third: ({string.Join(", ", third)})"); + TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + Succeeded = false; + } + } + + private void ValidateResult({Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] inArray2 = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] inArray3 = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray3[0]), op3); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* op3, void* result, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] inArray2 = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] inArray3 = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray3[0]), ref Unsafe.AsRef(op3), (uint)Unsafe.SizeOf<{Op1VectorType}<{Op1BaseType}>>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); + + ValidateResult(inArray1, inArray2, inArray3, outArray, method); + } + + private void ValidateResult({Op1BaseType}[] firstOp, {Op1BaseType}[] secondOp, {Op1BaseType}[] thirdOp, {RetBaseType}[] result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>, {Op1VectorType}<{Op1BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); + TestLibrary.TestFramework.LogInformation($"secondOp: ({string.Join(", ", secondOp)})"); + TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +} From 904285396db3c797bf4d92c37360906cc8554ea4 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 17 Jul 2024 13:56:06 -0400 Subject: [PATCH 2/2] Comment --- src/coreclr/jit/lsraarm64.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index a4ed3ac6cac53..cd4a5cdf69b4f 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1535,6 +1535,14 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou break; case NI_Sve_MultiplyAddRotateComplexBySelectedScalar: + // This API has two immediates, one of which is used to index pairs of floats in a vector. + // For a vector width of 128 bits, this means the index's range is [0, 1], + // which means we will skip the above jump table register check, + // even though we might need a jump table for the second immediate. + // Thus, this API is special-cased, and does not use the HW_Category_SIMDByIndexedElement path. + // Also, only one internal register is needed for the jump table; + // we will combine the two immediates into one jump table. + // Can only avoid generating a table if both immediates are constant. assert(intrin.op4->isContainedIntOrIImmed() == intrin.op5->isContainedIntOrIImmed()); needBranchTargetReg = !intrin.op4->isContainedIntOrIImmed();