Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ARM64-SVE: Add GatherVectorWithByteOffsetFirstFaulting #105369

Closed
wants to merge 30 commits into from
Closed

ARM64-SVE: Add GatherVectorWithByteOffsetFirstFaulting #105369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
d0efc9e
Initial work
TIHan Jul 2, 2024
42148fd
Merge remote-tracking branch 'upstream/main' into sve-ffr-part1
TIHan Jul 2, 2024
a7773ac
FirstFaulting partially works
TIHan Jul 2, 2024
76b42bd
Added template
TIHan Jul 2, 2024
bb01e37
Trying to test first-faulting behavior
TIHan Jul 4, 2024
a602b24
Using BoundedMemory to test FirstFaulting behavior for LoadVector.
TIHan Jul 6, 2024
60d410a
Fix size in validation
TIHan Jul 6, 2024
aee87d7
Added more helper functions. Added conditional select tests for LoadV…
TIHan Jul 8, 2024
7f3bb3c
Added first-faulting behavior tests for GatherVectorFirstFaulting
TIHan Jul 8, 2024
d952ff1
Merging with main
TIHan Jul 8, 2024
3923946
Added GetFfr suffix-style APIs
TIHan Jul 8, 2024
461b6a3
Fixing GatherVector tests
TIHan Jul 8, 2024
d5b8675
Formatting
TIHan Jul 8, 2024
07833e3
Feedback
TIHan Jul 9, 2024
ce5a9bd
Merge remote-tracking branch 'upstream/main' into sve-ffr-part1
TIHan Jul 9, 2024
05fb46d
Feedback
TIHan Jul 9, 2024
c63f878
Ensure the P/Invokes are blittable
tannergooding Jul 10, 2024
a4533fe
Merging
TIHan Jul 11, 2024
72d1dea
Merge remote-tracking branch 'upstream/main' into sve-ffr-part1
TIHan Jul 12, 2024
6c28927
Fix build
TIHan Jul 13, 2024
fb2012e
Remove checking for zeroes after the fault
TIHan Jul 13, 2024
aca6759
Added GatherVectorFirstFaultingVectorBases test template, but current…
TIHan Jul 16, 2024
d781fdc
Mark GetFfr methods as side-effectful
TIHan Jul 16, 2024
a73fe35
Verifying expected fault result. Test weaks.
TIHan Jul 19, 2024
81882a4
Merging with main
TIHan Jul 19, 2024
ad5ec2e
Fix build
TIHan Jul 20, 2024
a8919e3
Add GatherVectorWithByteOffsetFirstFaulting
amanasifkhalid Jul 22, 2024
7fc3b21
Fix tests
amanasifkhalid Jul 23, 2024
7ac2787
Remove log
amanasifkhalid Jul 23, 2024
aa8b720
Fix validation
amanasifkhalid Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/coreclr/jit/fgdiagnostic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3436,6 +3436,15 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block)
case NI_Sve_GatherPrefetch32Bit:
case NI_Sve_GatherPrefetch64Bit:
case NI_Sve_GatherPrefetch8Bit:
case NI_Sve_SetFfr:
case NI_Sve_GetFfrByte:
case NI_Sve_GetFfrInt16:
case NI_Sve_GetFfrInt32:
case NI_Sve_GetFfrInt64:
case NI_Sve_GetFfrSByte:
case NI_Sve_GetFfrUInt16:
case NI_Sve_GetFfrUInt32:
case NI_Sve_GetFfrUInt64:
{
assert(tree->OperRequiresCallFlag(this));
expectedFlags |= GTF_GLOB_REF;
Expand Down
32 changes: 27 additions & 5 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26548,6 +26548,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
break;

case NI_Sve_GatherVectorWithByteOffsets:
case NI_Sve_GatherVectorWithByteOffsetFirstFaulting:
case NI_Sve_LoadVector:
case NI_Sve_LoadVectorNonTemporal:
case NI_Sve_LoadVector128AndReplicateToVector:
Expand All @@ -26557,6 +26558,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
case NI_Sve_LoadVectorByteZeroExtendToUInt16:
case NI_Sve_LoadVectorByteZeroExtendToUInt32:
case NI_Sve_LoadVectorByteZeroExtendToUInt64:
case NI_Sve_LoadVectorFirstFaulting:
case NI_Sve_LoadVectorInt16SignExtendToInt32:
case NI_Sve_LoadVectorInt16SignExtendToInt64:
case NI_Sve_LoadVectorInt16SignExtendToUInt32:
Expand All @@ -26583,6 +26585,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const

case NI_Sve_GatherVector:
case NI_Sve_GatherVectorByteZeroExtend:
case NI_Sve_GatherVectorFirstFaulting:
case NI_Sve_GatherVectorInt16SignExtend:
case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend:
case NI_Sve_GatherVectorInt32SignExtend:
Expand Down Expand Up @@ -26674,11 +26677,12 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const
{
#ifdef TARGET_ARM64
static_assert_no_msg(
AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorInt16SignExtend,
NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend, NI_Sve_GatherVectorInt32SignExtend,
NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend, NI_Sve_GatherVectorSByteSignExtend,
NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt16ZeroExtend,
NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend, NI_Sve_GatherVectorUInt32ZeroExtend));
AreContiguous(NI_Sve_GatherVector, NI_Sve_GatherVectorByteZeroExtend, NI_Sve_GatherVectorFirstFaulting,
NI_Sve_GatherVectorInt16SignExtend, NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend,
NI_Sve_GatherVectorInt32SignExtend, NI_Sve_GatherVectorInt32WithByteOffsetsSignExtend,
NI_Sve_GatherVectorSByteSignExtend, NI_Sve_GatherVectorUInt16WithByteOffsetsZeroExtend,
NI_Sve_GatherVectorUInt16ZeroExtend, NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend,
NI_Sve_GatherVectorUInt32ZeroExtend));
assert(varTypeIsI(addr) || (varTypeIsSIMD(addr) && ((intrinsicId >= NI_Sve_GatherVector) &&
(intrinsicId <= NI_Sve_GatherVectorUInt32ZeroExtend))));
#else
Expand Down Expand Up @@ -27096,6 +27100,15 @@ bool GenTreeHWIntrinsic::OperRequiresCallFlag() const
case NI_Sve_GatherPrefetch32Bit:
case NI_Sve_GatherPrefetch64Bit:
case NI_Sve_GatherPrefetch8Bit:
case NI_Sve_SetFfr:
case NI_Sve_GetFfrByte:
case NI_Sve_GetFfrInt16:
case NI_Sve_GetFfrInt32:
case NI_Sve_GetFfrInt64:
case NI_Sve_GetFfrSByte:
case NI_Sve_GetFfrUInt16:
case NI_Sve_GetFfrUInt32:
case NI_Sve_GetFfrUInt64:
{
return true;
}
Expand Down Expand Up @@ -27286,6 +27299,15 @@ void GenTreeHWIntrinsic::Initialize(NamedIntrinsic intrinsicId)
case NI_Sve_GatherPrefetch32Bit:
case NI_Sve_GatherPrefetch64Bit:
case NI_Sve_GatherPrefetch8Bit:
case NI_Sve_SetFfr:
case NI_Sve_GetFfrByte:
case NI_Sve_GetFfrInt16:
case NI_Sve_GetFfrInt32:
case NI_Sve_GetFfrInt64:
case NI_Sve_GetFfrSByte:
case NI_Sve_GetFfrUInt16:
case NI_Sve_GetFfrUInt32:
case NI_Sve_GetFfrUInt64:
{
// Mark as a call and global reference, much as is done for GT_KEEPALIVE
gtFlags |= (GTF_CALL | GTF_GLOB_REF);
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/hwintrinsic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2217,6 +2217,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
#elif defined(TARGET_ARM64)
case NI_Sve_GatherVector:
case NI_Sve_GatherVectorByteZeroExtend:
case NI_Sve_GatherVectorFirstFaulting:
case NI_Sve_GatherVectorInt16SignExtend:
case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend:
case NI_Sve_GatherVectorInt32SignExtend:
Expand All @@ -2227,6 +2228,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend:
case NI_Sve_GatherVectorUInt32ZeroExtend:
case NI_Sve_GatherVectorWithByteOffsets:
case NI_Sve_GatherVectorWithByteOffsetFirstFaulting:
assert(varTypeIsSIMD(op3->TypeGet()));
if (numArgs == 3)
{
Expand Down
17 changes: 17 additions & 0 deletions src/coreclr/jit/hwintrinsiccodegenarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2051,6 +2051,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)

case NI_Sve_GatherVector:
case NI_Sve_GatherVectorByteZeroExtend:
case NI_Sve_GatherVectorFirstFaulting:
case NI_Sve_GatherVectorInt16SignExtend:
case NI_Sve_GatherVectorInt16WithByteOffsetsSignExtend:
case NI_Sve_GatherVectorInt32SignExtend:
Expand Down Expand Up @@ -2100,6 +2101,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
}

case NI_Sve_GatherVectorWithByteOffsets:
case NI_Sve_GatherVectorWithByteOffsetFirstFaulting:
{
assert(!varTypeIsSIMD(intrin.op2->gtType));
assert(intrin.numOperands == 3);
Expand Down Expand Up @@ -2366,6 +2368,21 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
break;
}

case NI_Sve_LoadVectorFirstFaulting:
{
assert(op3Reg == REG_NA);
insScalableOpts sopt = (opt == INS_OPTS_SCALABLE_B) ? INS_SCALABLE_OPTS_NONE : INS_SCALABLE_OPTS_LSL_N;
GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, REG_ZR, opt, sopt);
break;
}

case NI_Sve_SetFfr:
{
assert(targetReg == REG_NA);
GetEmitter()->emitIns_R(ins, emitSize, op1Reg, opt);
break;
}

case NI_Sve_ConditionalExtractAfterLastActiveElementScalar:
case NI_Sve_ConditionalExtractLastActiveElementScalar:
{
Expand Down
Loading
Loading