From bced5841eab7009ef9f4ef8590ffbe4845c1d68f Mon Sep 17 00:00:00 2001 From: Khushal Modi Date: Tue, 18 Jul 2023 14:18:10 -0700 Subject: [PATCH] Light up Ascii.Equality.Equals and Ascii.Equality.EqualsIgnoreCase with Vector512 code path (#88650) * merging with main Enabling AVX512 for ASCII.Equals * Correcting defects in the new Equals for AVX512 case * Correcting defects * Upgrading ASCII.Equality.EqualsIgnoreCase * Using intrinsics in AllCharsInVectorAreAscii * Using intrinsics in AllCharsInVectorAreAscii * Removing check for AVX512F and adding a check for Vector512 because the library is not using any functions from AVX512F * Removing check for CompExactlyDependsOn(AVX512F) from AllCharsInVectorAreAscii for Vector 512. Also checking for Vector512 support and not AVX512F in ASCIIEquality.Equals * Correcting the Tloader.Count512 for ushort * resolving merge errors * Adding TLoader method for Vector512 for EqualAndAscii * Updating Load512 for WideningLoader for performance increase * addressing review comments * Addressing review changes. Changing Widen to WidenLower for Load512 --- .../src/System/Text/Ascii.Equality.cs | 162 +++++++++++++++++- .../src/System/Text/Ascii.Utility.cs | 16 ++ 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs index 7b63ee518e060..31df908d9afb0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs @@ -61,6 +61,36 @@ private static bool Equals(ref TLeft left, ref TRight ri } } } + else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512.Count) + { + ref TLeft currentLeftSearchSpace = ref left; + ref TRight currentRightSearchSpace = ref right; + // Add Vector512.Count because TLeft == TRight + // Or we are in the Widen case where we iterate 2 * TRight.Count which is the same as TLeft.Count + Debug.Assert(Vector512.Count == Vector512.Count + || (typeof(TLoader) == typeof(WideningLoader) && Vector512.Count == Vector512.Count * 2)); + ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector512.Count); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + if (!TLoader.EqualAndAscii512(ref currentLeftSearchSpace, ref currentRightSearchSpace)) + { + return false; + } + + currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, Vector512.Count); + currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, Vector512.Count); + } + while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + + // If any elements remain, process the last vector in the search space. + if (length % (uint)Vector512.Count != 0) + { + ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector512.Count); + return TLoader.EqualAndAscii512(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd); + } + } else if (Avx.IsSupported && length >= (uint)Vector256.Count) { ref TLeft currentLeftSearchSpace = ref left; @@ -74,7 +104,7 @@ private static bool Equals(ref TLeft left, ref TRight ri // Loop until either we've finished all elements or there's less than a vector's-worth remaining. do { - if (!TLoader.EqualAndAscii(ref currentLeftSearchSpace, ref currentRightSearchSpace)) + if (!TLoader.EqualAndAscii256(ref currentLeftSearchSpace, ref currentRightSearchSpace)) { return false; } @@ -88,7 +118,7 @@ private static bool Equals(ref TLeft left, ref TRight ri if (length % (uint)Vector256.Count != 0) { ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref left, length - (uint)Vector256.Count); - return TLoader.EqualAndAscii(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd); + return TLoader.EqualAndAscii256(ref oneVectorAwayFromLeftEnd, ref oneVectorAwayFromRightEnd); } } else @@ -198,6 +228,77 @@ private static bool EqualsIgnoreCase(ref TLeft left, ref } } } + else if (Vector512.IsHardwareAccelerated && length >= (uint)Vector512.Count) + { + ref TLeft currentLeftSearchSpace = ref left; + ref TLeft oneVectorAwayFromLeftEnd = ref Unsafe.Add(ref currentLeftSearchSpace, length - TLoader.Count512); + ref TRight currentRightSearchSpace = ref right; + ref TRight oneVectorAwayFromRightEnd = ref Unsafe.Add(ref currentRightSearchSpace, length - (uint)Vector512.Count); + + Vector512 leftValues; + Vector512 rightValues; + + Vector512 loweringMask = Vector512.Create(TRight.CreateTruncating(0x20)); + Vector512 vecA = Vector512.Create(TRight.CreateTruncating('a')); + Vector512 vecZMinusA = Vector512.Create(TRight.CreateTruncating(('z' - 'a'))); + + // Loop until either we've finished all elements or there's less than a vector's-worth remaining. + do + { + leftValues = TLoader.Load512(ref currentLeftSearchSpace); + rightValues = Vector512.LoadUnsafe(ref currentRightSearchSpace); + if (!AllCharsInVectorAreAscii(leftValues | rightValues)) + { + return false; + } + + Vector512 notEquals = ~Vector512.Equals(leftValues, rightValues); + + if (notEquals != Vector512.Zero) + { + // not exact match + + leftValues |= loweringMask; + rightValues |= loweringMask; + + if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != rightValues) + { + return false; // first input isn't in [A-Za-z], and not exact match of lowered + } + } + + currentRightSearchSpace = ref Unsafe.Add(ref currentRightSearchSpace, (uint)Vector512.Count); + currentLeftSearchSpace = ref Unsafe.Add(ref currentLeftSearchSpace, TLoader.Count512); + } + while (!Unsafe.IsAddressGreaterThan(ref currentRightSearchSpace, ref oneVectorAwayFromRightEnd)); + + // If any elements remain, process the last vector in the search space. + if (length % (uint)Vector512.Count != 0) + { + leftValues = TLoader.Load512(ref oneVectorAwayFromLeftEnd); + rightValues = Vector512.LoadUnsafe(ref oneVectorAwayFromRightEnd); + + if (!AllCharsInVectorAreAscii(leftValues | rightValues)) + { + return false; + } + + Vector512 notEquals = ~Vector512.Equals(leftValues, rightValues); + + if (notEquals != Vector512.Zero) + { + // not exact match + + leftValues |= loweringMask; + rightValues |= loweringMask; + + if (Vector512.GreaterThanAny((leftValues - vecA) & notEquals, vecZMinusA) || leftValues != rightValues) + { + return false; // first input isn't in [A-Za-z], and not exact match of lowered + } + } + } + } else if (Avx.IsSupported && length >= (uint)Vector256.Count) { ref TLeft currentLeftSearchSpace = ref left; @@ -353,21 +454,26 @@ private interface ILoader { static abstract nuint Count128 { get; } static abstract nuint Count256 { get; } + static abstract nuint Count512 { get; } static abstract Vector128 Load128(ref TLeft ptr); static abstract Vector256 Load256(ref TLeft ptr); - static abstract bool EqualAndAscii(ref TLeft left, ref TRight right); + static abstract Vector512 Load512(ref TLeft ptr); + static abstract bool EqualAndAscii256(ref TLeft left, ref TRight right); + static abstract bool EqualAndAscii512(ref TLeft left, ref TRight right); } private readonly struct PlainLoader : ILoader where T : unmanaged, INumberBase { public static nuint Count128 => (uint)Vector128.Count; public static nuint Count256 => (uint)Vector256.Count; + public static nuint Count512 => (uint)Vector512.Count; public static Vector128 Load128(ref T ptr) => Vector128.LoadUnsafe(ref ptr); public static Vector256 Load256(ref T ptr) => Vector256.LoadUnsafe(ref ptr); + public static Vector512 Load512(ref T ptr) => Vector512.LoadUnsafe(ref ptr); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx))] - public static bool EqualAndAscii(ref T left, ref T right) + public static bool EqualAndAscii256(ref T left, ref T right) { Vector256 leftValues = Vector256.LoadUnsafe(ref left); Vector256 rightValues = Vector256.LoadUnsafe(ref right); @@ -379,12 +485,27 @@ public static bool EqualAndAscii(ref T left, ref T right) return true; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool EqualAndAscii512(ref T left, ref T right) + { + Vector512 leftValues = Vector512.LoadUnsafe(ref left); + Vector512 rightValues = Vector512.LoadUnsafe(ref right); + + if (leftValues != rightValues || !AllCharsInVectorAreAscii(leftValues)) + { + return false; + } + + return true; + } } private readonly struct WideningLoader : ILoader { public static nuint Count128 => sizeof(long); public static nuint Count256 => (uint)Vector128.Count; + public static nuint Count512 => (uint)Vector256.Count; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Load128(ref byte ptr) @@ -412,9 +533,15 @@ public static Vector256 Load256(ref byte ptr) return Vector256.Create(lower, upper); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Load512(ref byte ptr) + { + return Vector512.WidenLower(Vector256.LoadUnsafe(ref ptr).ToVector512()); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx))] - public static bool EqualAndAscii(ref byte utf8, ref ushort utf16) + public static bool EqualAndAscii256(ref byte utf8, ref ushort utf16) { // We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we search Debug.Assert(Vector256.Count == Vector256.Count * 2); @@ -437,6 +564,31 @@ public static bool EqualAndAscii(ref byte utf8, ref ushort utf16) return true; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool EqualAndAscii512(ref byte utf8, ref ushort utf16) + { + // We widen the utf8 param so we can compare it to utf16, this doubles how much of the utf16 vector we search + Debug.Assert(Vector512.Count == Vector512.Count * 2); + + Vector512 leftNotWidened = Vector512.LoadUnsafe(ref utf8); + if (!AllCharsInVectorAreAscii(leftNotWidened)) + { + return false; + } + + (Vector512 leftLower, Vector512 leftUpper) = Vector512.Widen(leftNotWidened); + Vector512 right = Vector512.LoadUnsafe(ref utf16); + Vector512 rightNext = Vector512.LoadUnsafe(ref utf16, (uint)Vector512.Count); + + // A branchless version of "leftLower != right || leftUpper != rightNext" + if (((leftLower ^ right) | (leftUpper ^ rightNext)) != Vector512.Zero) + { + return false; + } + + return true; + } } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs index e30b3d06c7810..45b73be6a618d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs @@ -1972,6 +1972,22 @@ private static bool AllCharsInVectorAreAscii(Vector256 vector) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AllCharsInVectorAreAscii(Vector512 vector) + where T : unmanaged + { + Debug.Assert(typeof(T) == typeof(byte) || typeof(T) == typeof(ushort)); + + if (typeof(T) == typeof(byte)) + { + return vector.AsByte().ExtractMostSignificantBits() == 0; + } + else + { + return (vector.AsUInt16() & Vector512.Create((ushort)0xFF80)) == Vector512.Zero; + } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ExtractAsciiVector(Vector128 vectorFirst, Vector128 vectorSecond) {