Skip to content

Commit

Permalink
Light up Ascii.Equality.Equals and Ascii.Equality.EqualsIgnoreCase wi…
Browse files Browse the repository at this point in the history 
…th Vector512 code path (#88650)

* merging with main
Enabling AVX512 for ASCII.Equals

* Correcting defects in the new Equals for AVX512 case

* Correcting defects

* Upgrading ASCII.Equality.EqualsIgnoreCase

* Using intrinsics in AllCharsInVectorAreAscii

* Using intrinsics in AllCharsInVectorAreAscii

* Removing check for AVX512F and adding a check for Vector512 because the library is not using any functions from AVX512F

* Removing check for CompExactlyDependsOn(AVX512F) from AllCharsInVectorAreAscii for Vector 512. Also checking for Vector512 support and not AVX512F in ASCIIEquality.Equals

* Correcting the Tloader.Count512 for ushort

* resolving merge errors

* Adding TLoader method for Vector512 for EqualAndAscii

* Updating Load512 for WideningLoader for performance increase

* addressing review comments

* Addressing review changes. Changing Widen to WidenLower for Load512
  • Loading branch information
@khushal1996
khushal1996 authored Jul 18, 2023
1 parent 420dd4e commit bced584
Show file tree
Hide file tree
Showing 2 changed files with 173 additions and 5 deletions.
162 changes: 157 additions & 5 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,36 @@ private static bool Equals<TLeft, TRight, TLoader>(ref TLeft left, ref TRight ri
}
}
}
else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512<TLeft>.Count)
{
ref TLeft currentLeftSearchSpace = ref left;
ref TRight currentRightSearchSpace = ref right;
// Add Vector512<TLeft>.Count because TLeft == TRight
// Or we are in the Widen case where we iterate 2 * TRight.Count which is the same as TLeft.Count
Debug.Assert(Vector512<TLeft>.Count == Vector512<TRight>.Count
|| (typeof(TLoader) == typeof(WideningLoader) && Vector512<TLeft>.Count == Vector512<TRight>.Count * 2));
ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector512<TLeft>.Count);

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
if (!TLoader.EqualAndAscii512(ref currentLeftSearchSpace, ref currentRightSearchSpace))
{
return false;
}

currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector512<TLeft>.Count);
currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector512<TLeft>.Count);
}
while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));

// If any elements remain, process the last vector in the search space.
if (length % (uint)Vector512<TLeft>.Count != 0)
{
ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector512<TLeft>.Count);
return TLoader.EqualAndAscii512(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd);
}
}
else if (Avx.IsSupported && length >= (uint)Vector256<TLeft>.Count)
{
ref TLeft currentLeftSearchSpace = ref left;
Expand All @@ -74,7 +104,7 @@ private static bool Equals<TLeft, TRight, TLoader>(ref TLeft left, ref TRight ri
// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
if (!TLoader.EqualAndAscii(ref currentLeftSearchSpace, ref currentRightSearchSpace))
if (!TLoader.EqualAndAscii256(ref currentLeftSearchSpace, ref currentRightSearchSpace))
{
return false;
}
Expand All @@ -88,7 +118,7 @@ private static bool Equals<TLeft, TRight, TLoader>(ref TLeft left, ref TRight ri
if (length % (uint)Vector256<TLeft>.Count != 0)
{
ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector256<TLeft>.Count);
return TLoader.EqualAndAscii(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd);
return TLoader.EqualAndAscii256(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd);
}
}
else
Expand Down Expand Up @@ -198,6 +228,77 @@ private static bool EqualsIgnoreCase<TLeft, TRight, TLoader>(ref TLeft left, ref
}
}
}
else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512<TRight>.Count)
{
ref TLeft currentLeftSearchSpace = ref left;
ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - TLoader.Count512);
ref TRight currentRightSearchSpace = ref right;
ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector512<TRight>.Count);

Vector512<TRight> leftValues;
Vector512<TRight> rightValues;

Vector512<TRight> loweringMask = Vector512.Create(TRight.CreateTruncating(0x20));
Vector512<TRight> vecA = Vector512.Create(TRight.CreateTruncating('a'));
Vector512<TRight> vecZMinusA = Vector512.Create(TRight.CreateTruncating(('z' - 'a')));

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
leftValues = TLoader.Load512(ref currentLeftSearchSpace);
rightValues = Vector512.LoadUnsafe(ref currentRightSearchSpace);
if (!AllCharsInVectorAreAscii(leftValues | rightValues))
{
return false;
}

Vector512<TRight> notEquals = ~Vector512.Equals(leftValues, rightValues);

if (notEquals != Vector512<TRight>.Zero)
{
// not exact match

leftValues |= loweringMask;
rightValues |= loweringMask;

if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != rightValues)
{
return false; // first input isn't in [A-Za-z], and not exact match of lowered
}
}

currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector512<TRight>.Count);
currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count512);
}
while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd));

// If any elements remain, process the last vector in the search space.
if (length % (uint)Vector512<TRight>.Count != 0)
{
leftValues = TLoader.Load512(ref oneVectorAwayFromLeftEnd);
rightValues = Vector512.LoadUnsafe(ref oneVectorAwayFromRightEnd);

if (!AllCharsInVectorAreAscii(leftValues | rightValues))
{
return false;
}

Vector512<TRight> notEquals = ~Vector512.Equals(leftValues, rightValues);

if (notEquals != Vector512<TRight>.Zero)
{
// not exact match

leftValues |= loweringMask;
rightValues |= loweringMask;

if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != rightValues)
{
return false; // first input isn't in [A-Za-z], and not exact match of lowered
}
}
}
}
else if (Avx.IsSupported && length >= (uint)Vector256<TRight>.Count)
{
ref TLeft currentLeftSearchSpace = ref left;
Expand Down Expand Up @@ -353,21 +454,26 @@ private interface ILoader<TLeft, TRight>
{
static abstract nuint Count128 { get; }
static abstract nuint Count256 { get; }
static abstract nuint Count512 { get; }
static abstract Vector128<TRight> Load128(ref TLeft ptr);
static abstract Vector256<TRight> Load256(ref TLeft ptr);
static abstract bool EqualAndAscii(ref TLeft left, ref TRight right);
static abstract Vector512<TRight> Load512(ref TLeft ptr);
static abstract bool EqualAndAscii256(ref TLeft left, ref TRight right);
static abstract bool EqualAndAscii512(ref TLeft left, ref TRight right);
}

private readonly struct PlainLoader<T> : ILoader<T, T> where T : unmanaged, INumberBase<T>
{
public static nuint Count128 => (uint)Vector128<T>.Count;
public static nuint Count256 => (uint)Vector256<T>.Count;
public static nuint Count512 => (uint)Vector512<T>.Count;
public static Vector128<T> Load128(ref T ptr) => Vector128.LoadUnsafe(ref ptr);
public static Vector256<T> Load256(ref T ptr) => Vector256.LoadUnsafe(ref ptr);
public static Vector512<T> Load512(ref T ptr) => Vector512.LoadUnsafe(ref ptr);

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Avx))]
public static bool EqualAndAscii(ref T left, ref T right)
public static bool EqualAndAscii256(ref T left, ref T right)
{
Vector256<T> leftValues = Vector256.LoadUnsafe(ref left);
Vector256<T> rightValues = Vector256.LoadUnsafe(ref right);
Expand All @@ -379,12 +485,27 @@ public static bool EqualAndAscii(ref T left, ref T right)

return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool EqualAndAscii512(ref T left, ref T right)
{
Vector512<T> leftValues = Vector512.LoadUnsafe(ref left);
Vector512<T> rightValues = Vector512.LoadUnsafe(ref right);

if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues))
{
return false;
}

return true;
}
}

private readonly struct WideningLoader : ILoader<byte, ushort>
{
public static nuint Count128 => sizeof(long);
public static nuint Count256 => (uint)Vector128<byte>.Count;
public static nuint Count512 => (uint)Vector256<byte>.Count;

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<ushort> Load128(ref byte ptr)
Expand Down Expand Up @@ -412,9 +533,15 @@ public static Vector256<ushort> Load256(ref byte ptr)
return Vector256.Create(lower, upper);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector512<ushort> Load512(ref byte ptr)
{
return Vector512.WidenLower(Vector256.LoadUnsafe(ref ptr).ToVector512());
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
[CompExactlyDependsOn(typeof(Avx))]
public static bool EqualAndAscii(ref byte utf8, ref ushort utf16)
public static bool EqualAndAscii256(ref byte utf8, ref ushort utf16)
{
// We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we search
Debug.Assert(Vector256<byte>.Count == Vector256<ushort>.Count * 2);
Expand All @@ -437,6 +564,31 @@ public static bool EqualAndAscii(ref byte utf8, ref ushort utf16)

return true;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool EqualAndAscii512(ref byte utf8, ref ushort utf16)
{
// We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we search
Debug.Assert(Vector512<byte>.Count == Vector512<ushort>.Count * 2);

Vector512<byte> leftNotWidened = Vector512.LoadUnsafe(ref utf8);
if (!AllCharsInVectorAreAscii(leftNotWidened))
{
return false;
}

(Vector512<ushort> leftLower, Vector512<ushort> leftUpper) = Vector512.Widen(leftNotWidened);
Vector512<ushort> right = Vector512.LoadUnsafe(ref utf16);
Vector512<ushort> rightNext = Vector512.LoadUnsafe(ref utf16, (uint)Vector512<ushort>.Count);

// A branchless version of "leftLower != right || leftUpper != rightNext"
if (((leftLower ^ right) | (leftUpper ^ rightNext)) != Vector512<ushort>.Zero)
{
return false;
}

return true;
}
}
}
}
16 changes: 16 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1972,6 +1972,22 @@ private static bool AllCharsInVectorAreAscii<T>(Vector256<T> vector)
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AllCharsInVectorAreAscii<T>(Vector512<T> vector)
where T : unmanaged
{
Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort));

if (typeof(T) == typeof(byte))
{
return vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return (vector.AsUInt16() & Vector512.Create((ushort)0xFF80)) == Vector512<ushort>.Zero;
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ExtractAsciiVector(Vector128<ushort> vectorFirst, Vector128<ushort> vectorSecond)
{
Expand Down

0 comments on commit bced584

Please sign in to comment.