From 1566d85f719e8bb6b5cd0af3c628e9ce621c5a82 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Tue, 4 Jul 2023 23:31:27 +0200 Subject: [PATCH 01/43] Add SearchValues --- .../System.Memory/ref/System.Memory.cs | 4 + .../tests/Span/StringSearchValues.cs | 450 ++++++++++++++ .../tests/System.Memory.Tests.csproj | 13 +- .../src/Resources/Strings.resx | 3 + .../System.Private.CoreLib.Shared.projitems | 19 + .../src/System/Globalization/Ordinal.cs | 2 +- .../src/System/Globalization/TextInfo.cs | 2 +- .../src/System/MemoryExtensions.cs | 39 ++ .../System/SearchValues/EmptySearchValues.cs | 3 + .../SearchValues/IndexOfAnyAsciiSearcher.cs | 25 +- .../System/SearchValues/ProbabilisticMap.cs | 3 +- .../src/System/SearchValues/SearchValues.T.cs | 6 +- .../src/System/SearchValues/SearchValues.cs | 22 +- .../AsciiStringSearchValuesTeddyBase.cs | 546 +++++++++++++++++ ...sciiStringSearchValuesTeddyBucketizedN2.cs | 25 + ...sciiStringSearchValuesTeddyBucketizedN3.cs | 25 + ...iStringSearchValuesTeddyNonBucketizedN2.cs | 25 + ...iStringSearchValuesTeddyNonBucketizedN3.cs | 25 + .../Strings/Helpers/AhoCorasick.cs | 548 ++++++++++++++++++ .../Strings/Helpers/AhoCorasickBuilder.cs | 191 ++++++ .../Helpers/CharacterFrequencyHelper.cs | 127 ++++ .../Strings/Helpers/EightPackedReferences.cs | 48 ++ .../SearchValues/Strings/Helpers/RabinKarp.cs | 156 +++++ .../Helpers/StringSearchValuesHelper.cs | 294 ++++++++++ .../Strings/Helpers/TeddyBucketizer.cs | 134 +++++ .../Strings/Helpers/TeddyHelper.cs | 368 ++++++++++++ .../SingleStringSearchValuesFallback.cs | 26 + .../SingleStringSearchValuesThreeChars.cs | 328 +++++++++++ .../Strings/StringSearchValues.cs | 391 +++++++++++++ .../Strings/StringSearchValuesAhoCorasick.cs | 22 + .../Strings/StringSearchValuesBase.cs | 72 +++ .../Strings/StringSearchValuesRabinKarp.cs | 21 + .../src/System/SpanHelpers.Packed.cs | 12 +- .../src/System/Text/Ascii.Equality.cs | 4 + .../System.Runtime/ref/System.Runtime.cs | 1 + 35 files changed, 3940 insertions(+), 40 deletions(-) create mode 100644 src/libraries/System.Memory/tests/Span/StringSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/CharacterFrequencyHelper.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesFallback.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesAhoCorasick.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesBase.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesRabinKarp.cs diff --git a/src/libraries/System.Memory/ref/System.Memory.cs b/src/libraries/System.Memory/ref/System.Memory.cs index ce19bc6cbde03..5755daafedc55 100644 --- a/src/libraries/System.Memory/ref/System.Memory.cs +++ b/src/libraries/System.Memory/ref/System.Memory.cs @@ -234,6 +234,8 @@ public static partial class MemoryExtensions public static bool Contains(this System.ReadOnlySpan span, System.ReadOnlySpan value, System.StringComparison comparisonType) { throw null; } public static bool Contains(this System.ReadOnlySpan span, T value) where T : System.IEquatable? { throw null; } public static bool Contains(this System.Span span, T value) where T : System.IEquatable? { throw null; } + public static bool ContainsAny(this System.ReadOnlySpan span, System.Buffers.SearchValues values) { throw null; } + public static bool ContainsAny(this System.Span span, System.Buffers.SearchValues values) { throw null; } public static bool ContainsAny(this System.ReadOnlySpan span, System.Buffers.SearchValues values) where T : System.IEquatable? { throw null; } public static bool ContainsAny(this System.ReadOnlySpan span, System.ReadOnlySpan values) where T : System.IEquatable? { throw null; } public static bool ContainsAny(this System.ReadOnlySpan span, T value0, T value1) where T : System.IEquatable? { throw null; } @@ -271,6 +273,8 @@ public static void CopyTo(this T[]? source, System.Span destination) { } public static System.Text.SpanRuneEnumerator EnumerateRunes(this System.Span span) { throw null; } public static bool Equals(this System.ReadOnlySpan span, System.ReadOnlySpan other, System.StringComparison comparisonType) { throw null; } public static int IndexOf(this System.ReadOnlySpan span, System.ReadOnlySpan value, System.StringComparison comparisonType) { throw null; } + public static int IndexOfAny(this System.ReadOnlySpan span, System.Buffers.SearchValues values) { throw null; } + public static int IndexOfAny(this System.Span span, System.Buffers.SearchValues values) { throw null; } public static int IndexOfAny(this System.ReadOnlySpan span, System.Buffers.SearchValues values) where T : System.IEquatable? { throw null; } public static int IndexOfAny(this System.ReadOnlySpan span, System.ReadOnlySpan values) where T : System.IEquatable? { throw null; } public static int IndexOfAny(this System.ReadOnlySpan span, T value0, T value1) where T : System.IEquatable? { throw null; } diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs new file mode 100644 index 0000000000000..0b5d195d96363 --- /dev/null +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -0,0 +1,450 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; +using Xunit; + +namespace System.Memory.Tests.Span +{ + public static class StringSearchValuesTests + { + public static bool CanTestInvariantCulture => RemoteExecutor.IsSupported; + public static bool CanTestNls => RemoteExecutor.IsSupported && OperatingSystem.IsWindows(); + + [Theory] + [InlineData(StringComparison.Ordinal, "a", "ab", "abc", "bc")] + [InlineData(StringComparison.Ordinal, "A", "ab", "aBc", "Bc")] + [InlineData(StringComparison.OrdinalIgnoreCase, "a", "Ab", "abc", "bC")] + public static void Values_ImplementsSearchValuesBase(StringComparison comparisonType, params string[] values) + { + const string ValueNotInSet = "Hello world"; + + SearchValues stringValues = SearchValues.Create(values, comparisonType); + + Assert.False(stringValues.Contains(ValueNotInSet)); + + AssertIndexOfAnyAndFriends(Span.Empty, -1, -1, -1, -1); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet }, -1, 0, -1, 0); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, ValueNotInSet }, -1, 0, -1, 1); + + foreach (string value in values) + { + string differentCase = value.ToLowerInvariant(); + if (value == differentCase) + { + differentCase = value.ToUpperInvariant(); + Assert.NotEqual(value, differentCase); + } + + Assert.True(stringValues.Contains(value)); + Assert.Equal(comparisonType == StringComparison.OrdinalIgnoreCase, stringValues.Contains(differentCase)); + + AssertIndexOfAnyAndFriends(new[] { value }, 0, -1, 0, -1); + AssertIndexOfAnyAndFriends(new[] { value, value }, 0, -1, 1, -1); + AssertIndexOfAnyAndFriends(new[] { value, ValueNotInSet }, 0, 1, 0, 1); + AssertIndexOfAnyAndFriends(new[] { value, ValueNotInSet, ValueNotInSet }, 0, 1, 0, 2); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, value }, 1, 0, 1, 0); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, ValueNotInSet, value }, 2, 0, 2, 1); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, value, ValueNotInSet }, 1, 0, 1, 2); + AssertIndexOfAnyAndFriends(new[] { value, ValueNotInSet, value }, 0, 1, 2, 1); + + if (comparisonType == StringComparison.OrdinalIgnoreCase) + { + AssertIndexOfAnyAndFriends(new[] { differentCase }, 0, -1, 0, -1); + AssertIndexOfAnyAndFriends(new[] { differentCase, differentCase }, 0, -1, 1, -1); + AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet }, 0, 1, 0, 1); + AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet, ValueNotInSet }, 0, 1, 0, 2); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, differentCase }, 1, 0, 1, 0); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, ValueNotInSet, differentCase }, 2, 0, 2, 1); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, differentCase, ValueNotInSet }, 1, 0, 1, 2); + AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet, differentCase }, 0, 1, 2, 1); + } + else + { + AssertIndexOfAnyAndFriends(new[] { differentCase }, -1, 0, -1, 0); + AssertIndexOfAnyAndFriends(new[] { differentCase, differentCase }, -1, 0, -1, 1); + AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet }, -1, 0, -1, 1); + AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, differentCase }, -1, 0, -1, 1); + AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet, ValueNotInSet }, -1, 0, -1, 2); + } + } + + void AssertIndexOfAnyAndFriends(Span values, int any, int anyExcept, int last, int lastExcept) + { + Assert.Equal(any >= 0, last >= 0); + Assert.Equal(anyExcept >= 0, lastExcept >= 0); + + Assert.Equal(any, values.IndexOfAny(stringValues)); + Assert.Equal(any, ((ReadOnlySpan)values).IndexOfAny(stringValues)); + Assert.Equal(anyExcept, values.IndexOfAnyExcept(stringValues)); + Assert.Equal(anyExcept, ((ReadOnlySpan)values).IndexOfAnyExcept(stringValues)); + Assert.Equal(last, values.LastIndexOfAny(stringValues)); + Assert.Equal(last, ((ReadOnlySpan)values).LastIndexOfAny(stringValues)); + Assert.Equal(lastExcept, values.LastIndexOfAnyExcept(stringValues)); + Assert.Equal(lastExcept, ((ReadOnlySpan)values).LastIndexOfAnyExcept(stringValues)); + + Assert.Equal(any >= 0, values.ContainsAny(stringValues)); + Assert.Equal(any >= 0, ((ReadOnlySpan)values).ContainsAny(stringValues)); + Assert.Equal(anyExcept >= 0, values.ContainsAnyExcept(stringValues)); + Assert.Equal(anyExcept >= 0, ((ReadOnlySpan)values).ContainsAnyExcept(stringValues)); + } + } + + [Theory] + // Sets with empty values + [InlineData(StringComparison.Ordinal, 0, " ", "abc, ")] + [InlineData(StringComparison.OrdinalIgnoreCase, 0, " ", "abc, ")] + [InlineData(StringComparison.Ordinal, 0, "", "")] + [InlineData(StringComparison.OrdinalIgnoreCase, 0, "", "abc, ")] + // Empty sets + [InlineData(StringComparison.Ordinal, -1, " ", null)] + [InlineData(StringComparison.OrdinalIgnoreCase, -1, " ", null)] + [InlineData(StringComparison.Ordinal, -1, "", null)] + [InlineData(StringComparison.OrdinalIgnoreCase, -1, "", null)] + // Multiple potential matches - we want the first one + [InlineData(StringComparison.Ordinal, 1, "abcd", "bc, cd")] + // Simple case sensitivity + [InlineData(StringComparison.Ordinal, -1, " ABC", "abc")] + [InlineData(StringComparison.Ordinal, 1, " abc", "abc")] + [InlineData(StringComparison.OrdinalIgnoreCase, 1, " ABC", "abc")] + // A few more complex cases that test the Aho-Corasick implementation + [InlineData(StringComparison.Ordinal, 3, "RyrIGEdt2S9", "IGEdt2, G, rIGm6i")] + [InlineData(StringComparison.Ordinal, 2, "Npww1HtmO", "NVOhQu, w, XeR")] + [InlineData(StringComparison.Ordinal, 1, "08Qq6", "8, vx, BFA4s, aLP2, hm, lmT, y, CNTB, Q, vd")] + [InlineData(StringComparison.Ordinal, 3, "A4sRYUhKZR1Vn8N", "F, scsx, nWBhrx, Q, 7Of, BX, huoJ, R")] + [InlineData(StringComparison.Ordinal, 9, "40sufu3TdzcKQfK", "3MXvo26, zPd6t, zc, c5, ypUCK3A9, K, YlX")] + [InlineData(StringComparison.Ordinal, 0, "111KtTGeWuV", "11, B51tJ, Z, j0DWudC, kuJRbcovn, 0T2vnT9")] + [InlineData(StringComparison.Ordinal, 5, "Uykbt1zWw7wylEgC", "1zWw7, Bh, 7qDgAY, w, Z, dP, V, W, Hiols, T")] + [InlineData(StringComparison.Ordinal, 6, "PI9yZx9AOWrUR", "4, A, MLbg, jACE, x9AZEYPbLr, 4bYTzw, W, 9AOW, O")] + [InlineData(StringComparison.Ordinal, 7, "KV4cRyrIGEdt2S9kbXVK", "e64, 10Yw7k, IGEdt2, G, brL, rIGm6i, Z3, FHoVN, 7P2s")] + // OrdinalIgnoreCase does not match ASCII chars with non-ASCII ones + [InlineData(StringComparison.OrdinalIgnoreCase, 4, "AAAA\u212ABKBkBBCCCC", "\u212A")] + [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAAKB\u212ABkBBCCCC", "\u212A")] + [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAAkB\u212ABKBBCCCC", "\u212A")] + [InlineData(StringComparison.OrdinalIgnoreCase, 4, "AAAA\u017FBSBsBBCCCC", "\u017F")] + [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAASB\u017FBsBBCCCC", "\u017F")] + [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAAsB\u017FBSBBCCCC", "\u017F")] + public static void IndexOfAny(StringComparison comparisonType, int expected, string text, string? values) + { + Span textSpan = text.ToArray(); // Test non-readonly Span overloads + + string[] valuesArray = values is null ? Array.Empty() : values.Split(", "); + + SearchValues stringValues = SearchValues.Create(valuesArray, comparisonType); + + Assert.Equal(expected, text.AsSpan().IndexOfAny(stringValues)); + Assert.Equal(expected, textSpan.IndexOfAny(stringValues)); + + Assert.Equal(expected, IndexOfAnyReferenceImpl(text, valuesArray, comparisonType)); + + Assert.Equal(expected >= 0, text.AsSpan().ContainsAny(stringValues)); + Assert.Equal(expected >= 0, textSpan.ContainsAny(stringValues)); + } + + [Fact] + public static void Create_OnlyOrdinalComparisonIsSupported() + { + foreach (StringComparison comparisonType in Enum.GetValues()) + { + if (comparisonType is StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase) + { + _ = SearchValues.Create(new[] { "abc" }, comparisonType); + } + else + { + Assert.Throws(() => SearchValues.Create(new[] { "abc" }, comparisonType)); + } + } + } + + [Fact] + public static void Create_ThrowsOnNullValues() + { + Assert.Throws("values", () => SearchValues.Create(new[] { "foo", null, "bar" }, StringComparison.Ordinal)); + } + + [Fact] + public static void TestIndexOfAny_RandomInputs() + { + var helper = new StringSearchValuesTestHelper( + expected: IndexOfAnyReferenceImpl, + searchValues: (searchSpace, values) => searchSpace.IndexOfAny(values)); + + helper.TestRandomInputs(); + } + + [ConditionalFact(nameof(CanTestInvariantCulture))] + public static void TestIndexOfAny_RandomInputs_InvariantCulture() + { + RunUsingInvariantCulture(static () => + { + Assert.Equal("Invariant Language (Invariant Country)", CultureInfo.CurrentCulture.NativeName); + + TestIndexOfAny_RandomInputs(); + }); + } + + [ConditionalFact(nameof(CanTestNls))] + public static void TestIndexOfAny_RandomInputs_Nls() + { + RunUsingNLS(static () => + { + Assert.NotEqual("Invariant Language (Invariant Country)", CultureInfo.CurrentCulture.NativeName); + + TestIndexOfAny_RandomInputs(); + }); + } + + [Fact] + [ActiveIssue("Manual execution only. Worth running any time SearchValues logic is modified.")] + public static void TestIndexOfAny_RandomInputs_Stress() + { + RunStress(); + + if (CanTestInvariantCulture) + { + RunUsingInvariantCulture(static () => RunStress()); + } + + if (CanTestNls) + { + RunUsingNLS(static () => RunStress()); + } + + static void RunStress() + { + foreach (int maxNeedleCount in new[] { 2, 8, 20, 100 }) + { + foreach (int maxNeedleValueLength in new[] { 8, 40 }) + { + foreach (int haystackLength in new[] { 100, 1024 }) + { + var helper = new StringSearchValuesTestHelper( + expected: IndexOfAnyReferenceImpl, + searchValues: (searchSpace, values) => searchSpace.IndexOfAny(values)) + { + MaxNeedleCount = maxNeedleCount, + MaxNeedleValueLength = maxNeedleValueLength, + MaxHaystackLength = haystackLength, + HaystackIterationsPerNeedle = 1_000, + }; + + helper.StressRandomInputs(TimeSpan.FromSeconds(5)); + } + } + } + } + } + + private static int IndexOfAnyReferenceImpl(ReadOnlySpan searchSpace, ReadOnlySpan values, StringComparison comparisonType) + { + int minIndex = int.MaxValue; + + foreach (string value in values) + { + int i = searchSpace.IndexOf(value, comparisonType); + if ((uint)i < minIndex) + { + minIndex = i; + } + } + + return minIndex == int.MaxValue ? -1 : minIndex; + } + + private static void RunUsingInvariantCulture(Action action) + { + Assert.True(CanTestInvariantCulture); + + var psi = new ProcessStartInfo(); + psi.Environment.Clear(); + psi.Environment.Add("DOTNET_SYSTEM_GLOBALIZATION_INVARIANT", "true"); + + RemoteExecutor.Invoke(action, new RemoteInvokeOptions { StartInfo = psi, TimeOut = 10 * 60 * 1000 }).Dispose(); + } + + private static void RunUsingNLS(Action action) + { + Assert.True(CanTestNls); + + var psi = new ProcessStartInfo(); + psi.Environment.Clear(); + psi.Environment.Add("DOTNET_SYSTEM_GLOBALIZATION_USENLS", "true"); + + RemoteExecutor.Invoke(action, new RemoteInvokeOptions { StartInfo = psi, TimeOut = 10 * 60 * 1000 }).Dispose(); + } + + private sealed class StringSearchValuesTestHelper + { + public int MaxNeedleCount = 20; + public int MaxNeedleValueLength = 10; + public int MaxHaystackLength = 100; + public int HaystackIterationsPerNeedle = 50; + public int MinValueLength = 1; + + private static readonly char[] s_randomAsciiChars; + private static readonly char[] s_randomSimpleAsciiChars; + private static readonly char[] s_randomChars; + + static StringSearchValuesTestHelper() + { + s_randomAsciiChars = new char[100 * 1024]; + s_randomSimpleAsciiChars = new char[100 * 1024]; + s_randomChars = new char[1024 * 1024]; + + var rng = new Random(42); + + for (int i = 0; i < s_randomAsciiChars.Length; i++) + { + s_randomAsciiChars[i] = (char)rng.Next(0, 128); + } + + for (int i = 0; i < s_randomSimpleAsciiChars.Length; i++) + { + int random = rng.Next(26 * 2 + 10); + + s_randomSimpleAsciiChars[i] = (char)(random + (random switch + { + < 10 => '0', + < 36 => 'a' - 10, + _ => 'A' - 36, + })); + } + + rng.NextBytes(MemoryMarshal.Cast(s_randomChars)); + } + + private readonly IndexOfAnySearchDelegate _expectedDelegate; + private readonly SearchValuesSearchDelegate _searchValuesDelegate; + + public StringSearchValuesTestHelper(IndexOfAnySearchDelegate expected, SearchValuesSearchDelegate searchValues) + { + _expectedDelegate = expected; + _searchValuesDelegate = searchValues; + } + + public delegate int IndexOfAnySearchDelegate(ReadOnlySpan searchSpace, ReadOnlySpan values, StringComparison comparisonType); + + public delegate int SearchValuesSearchDelegate(ReadOnlySpan searchSpace, SearchValues values); + + public void StressRandomInputs(TimeSpan duration) + { + ExceptionDispatchInfo? exception = null; + Stopwatch s = Stopwatch.StartNew(); + + Parallel.For(0, Environment.ProcessorCount - 1, _ => + { + while (s.Elapsed < duration && Volatile.Read(ref exception) is null) + { + try + { + TestRandomInputs(iterationCount: 1, rng: new Random()); + } + catch (Exception ex) + { + exception = ExceptionDispatchInfo.Capture(ex); + } + } + }); + + exception?.Throw(); + } + + public void TestRandomInputs(int iterationCount = 1_000, Random? rng = null) + { + rng ??= new Random(42); + + for (int iterations = 0; iterations < iterationCount; iterations++) + { + // There are more interesting corner cases with ASCII needles, test those more. + Test(rng, s_randomSimpleAsciiChars, s_randomSimpleAsciiChars); + Test(rng, s_randomAsciiChars, s_randomSimpleAsciiChars); + Test(rng, s_randomSimpleAsciiChars, s_randomAsciiChars); + Test(rng, s_randomAsciiChars, s_randomAsciiChars); + Test(rng, s_randomChars, s_randomSimpleAsciiChars); + Test(rng, s_randomChars, s_randomAsciiChars); + + Test(rng, s_randomChars, s_randomChars); + } + } + + private void Test(Random rng, ReadOnlySpan haystackRandom, ReadOnlySpan needleRandom) + { + string[] values = new string[rng.Next(MaxNeedleCount) + 1]; + + for (int i = 0; i < values.Length; i++) + { + ReadOnlySpan valueSpan; + do + { + valueSpan = GetRandomSlice(rng, needleRandom, MaxNeedleValueLength); + } + while (valueSpan.Length < MinValueLength); + + values[i] = valueSpan.ToString(); + } + + SearchValues valuesOrdinal = SearchValues.Create(values, StringComparison.Ordinal); + SearchValues valuesOrdinalIgnoreCase = SearchValues.Create(values, StringComparison.OrdinalIgnoreCase); + + for (int i = 0; i < HaystackIterationsPerNeedle; i++) + { + Test(rng, StringComparison.Ordinal, haystackRandom, values, valuesOrdinal); + Test(rng, StringComparison.OrdinalIgnoreCase, haystackRandom, values, valuesOrdinalIgnoreCase); + } + } + + private void Test(Random rng, StringComparison comparisonType, ReadOnlySpan haystackRandom, + string[] needle, SearchValues searchValuesInstance) + { + ReadOnlySpan haystack = GetRandomSlice(rng, haystackRandom, MaxHaystackLength); + + int expectedIndex = _expectedDelegate(haystack, needle, comparisonType); + int searchValuesIndex = _searchValuesDelegate(haystack, searchValuesInstance); + + if (expectedIndex != searchValuesIndex) + { + AssertionFailed(haystack, needle, searchValuesInstance, comparisonType, expectedIndex, searchValuesIndex); + } + } + + private static ReadOnlySpan GetRandomSlice(Random rng, ReadOnlySpan span, int maxLength) + { + ReadOnlySpan slice = span.Slice(rng.Next(span.Length + 1)); + return slice.Slice(0, Math.Min(slice.Length, rng.Next(maxLength + 1))); + } + + private static void AssertionFailed(ReadOnlySpan haystack, string[] needle, SearchValues searchValues, StringComparison comparisonType, int expected, int actual) + { + Type implType = searchValues.GetType(); + string impl = $"{implType.Name} [{string.Join(", ", implType.GenericTypeArguments.Select(t => t.Name))}]"; + + string readableHaystack = ReadableAsciiOrSerialized(haystack.ToString()); + string readableNeedle = string.Join(", ", needle.Select(ReadableAsciiOrSerialized)); + + Assert.True(false, $"Expected {expected}, got {actual} for impl='{impl}' comparison={comparisonType} needle='{readableNeedle}', haystack='{readableHaystack}'"); + + static string ReadableAsciiOrSerialized(string value) + { + foreach (char c in value) + { + if (!char.IsAsciiLetterOrDigit(c)) + { + return $"[ {string.Join(", ", value.Select(c => int.CreateChecked(c)))} ]"; + } + } + + return value; + } + } + } + } +} diff --git a/src/libraries/System.Memory/tests/System.Memory.Tests.csproj b/src/libraries/System.Memory/tests/System.Memory.Tests.csproj index ca10d25a3872c..05e7c60c17bb8 100644 --- a/src/libraries/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/libraries/System.Memory/tests/System.Memory.Tests.csproj @@ -18,14 +18,13 @@ - + - + + @@ -276,9 +275,7 @@ - - + + diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index bd016b716a161..91da334936796 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -4244,4 +4244,7 @@ String length exceeded supported range. + + SearchValues<string> supports only Ordinal and OrdinalIgnoreCase StringComparison semantics. + diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 82eb84609a6f0..32e726a203cc9 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -437,6 +437,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 36854cd07bab7..3c5a38ac17034 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -78,7 +78,7 @@ internal static int CompareStringIgnoreCaseNonAscii(ref char strA, int lengthA, return OrdinalCasing.CompareStringIgnoreCase(ref strA, lengthA, ref strB, lengthB); } - private static bool EqualsIgnoreCase_Vector128(ref char charA, ref char charB, int length) + internal static bool EqualsIgnoreCase_Vector128(ref char charA, ref char charB, int length) { Debug.Assert(length >= Vector128.Count); Debug.Assert(Vector128.IsHardwareAccelerated); diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs index 28b9458abed47..eca1831c625bf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs @@ -436,7 +436,7 @@ public string ToUpper(string str) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static char ToUpperAsciiInvariant(char c) + internal static char ToUpperAsciiInvariant(char c) { if (char.IsAsciiLetterLower(c)) { diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 9153dc2d9e2fd..17804a753ef6b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -416,6 +416,11 @@ public static bool ContainsAny(this Span span, ReadOnlySpan values) whe public static bool ContainsAny(this Span span, SearchValues values) where T : IEquatable? => ContainsAny((ReadOnlySpan)span, values); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool ContainsAny(this Span span, SearchValues values) => + ContainsAny((ReadOnlySpan)span, values); + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool ContainsAnyExcept(this Span span, T value) where T : IEquatable? => @@ -490,6 +495,15 @@ public static bool ContainsAny(this ReadOnlySpan span, ReadOnlySpan val public static bool ContainsAny(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => IndexOfAny(span, values) >= 0; + /// + /// Searches for any occurance of any of the specified substring and returns true if found. If not found, returns false. + /// + /// The span to search. + /// The set of values to search for. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool ContainsAny(this ReadOnlySpan span, SearchValues values) => + IndexOfAny(span, values) >= 0; + /// /// Searches for any value other than the specified . /// @@ -1872,6 +1886,15 @@ public static int IndexOfAny(this Span span, ReadOnlySpan values) where public static int IndexOfAny(this Span span, SearchValues values) where T : IEquatable? => IndexOfAny((ReadOnlySpan)span, values); + /// + /// Searches for the first index of any of the specified substring values. + /// + /// The span to search. + /// The set of values to search for. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAny(this Span span, SearchValues values) => + IndexOfAny((ReadOnlySpan)span, values); + /// /// Searches for the first index of any of the specified values similar to calling IndexOf several times with the logical OR operator. If not found, returns -1. /// @@ -2061,6 +2084,22 @@ public static unsafe int IndexOfAny(this ReadOnlySpan span, ReadOnlySpan(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => SearchValues.IndexOfAny(span, values); + /// + /// Searches for the first index of any of the specified substring values. + /// + /// The span to search. + /// The set of values to search for. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int IndexOfAny(this ReadOnlySpan span, SearchValues values) + { + if (values is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); + } + + return values.IndexOfAnyMultiString(span); + } + /// /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1. /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/EmptySearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/EmptySearchValues.cs index c5f95097c011e..98836b0ead270 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/EmptySearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/EmptySearchValues.cs @@ -23,5 +23,8 @@ internal override int LastIndexOfAny(ReadOnlySpan span) => internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => span.Length - 1; + + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + -1; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs index 4f81a9b19970c..f98d6a580d487 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs @@ -19,6 +19,11 @@ internal static class IndexOfAnyAsciiSearcher { internal static bool IsVectorizationSupported => Ssse3.IsSupported || AdvSimd.Arm64.IsSupported || PackedSimd.IsSupported; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool BitmapContains(ref Vector256 bitmap, char c) => + c <= 127 && + (bitmap.GetElementUnsafe(c & 0xF) & (1 << (c >> 4))) != 0; + internal static unsafe void ComputeBitmap256(ReadOnlySpan values, out Vector256 bitmap0, out Vector256 bitmap1, out BitVector256 lookup) { // The exact format of these bitmaps differs from the other ComputeBitmap overloads as it's meant for the full [0, 255] range algorithm. @@ -1022,7 +1027,7 @@ private static unsafe int ComputeFirstIndex(ref T searchSpace, ref { if (typeof(T) == typeof(short)) { - result = FixUpPackedVector256Result(result); + result = PackedSpanHelpers.FixUpPackedVector256Result(result); } uint mask = TNegator.ExtractMask(result); @@ -1038,7 +1043,7 @@ private static unsafe int ComputeFirstIndexOverlapped(ref T searchS { if (typeof(T) == typeof(short)) { - result = FixUpPackedVector256Result(result); + result = PackedSpanHelpers.FixUpPackedVector256Result(result); } uint mask = TNegator.ExtractMask(result); @@ -1060,7 +1065,7 @@ private static unsafe int ComputeLastIndex(ref T searchSpace, ref T { if (typeof(T) == typeof(short)) { - result = FixUpPackedVector256Result(result); + result = PackedSpanHelpers.FixUpPackedVector256Result(result); } uint mask = TNegator.ExtractMask(result); @@ -1076,7 +1081,7 @@ private static unsafe int ComputeLastIndexOverlapped(ref T searchSp { if (typeof(T) == typeof(short)) { - result = FixUpPackedVector256Result(result); + result = PackedSpanHelpers.FixUpPackedVector256Result(result); } uint mask = TNegator.ExtractMask(result); @@ -1091,18 +1096,6 @@ private static unsafe int ComputeLastIndexOverlapped(ref T searchSp return offsetInVector - Vector256.Count + (int)((nuint)Unsafe.ByteOffset(ref searchSpace, ref secondVector) / (nuint)sizeof(T)); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - private static Vector256 FixUpPackedVector256Result(Vector256 result) - { - Debug.Assert(Avx2.IsSupported); - // Avx2.PackUnsignedSaturate(Vector256.Create((short)1), Vector256.Create((short)2)) will result in - // 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 - // We want to swap the X and Y bits - // 1, 1, 1, 1, 1, 1, 1, 1, X, X, X, X, X, X, X, X, Y, Y, Y, Y, Y, Y, Y, Y, 2, 2, 2, 2, 2, 2, 2, 2 - return Avx2.Permute4x64(result.AsInt64(), 0b_11_01_10_00).AsByte(); - } - internal interface INegator { static abstract bool NegateIfNeeded(bool result); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs index f42e130afeb43..bfad77993dbac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs @@ -365,8 +365,7 @@ private static int IndexOfAnyVectorized(ref uint charMap, ref char searchSpace, if (result != Vector256.Zero) { - // Account for how ContainsMask32CharsAvx2 packed the source chars (Avx2.PackUnsignedSaturate). - result = Avx2.Permute4x64(result.AsInt64(), 0b_11_01_10_00).AsByte(); + result = PackedSpanHelpers.FixUpPackedVector256Result(result); uint mask = result.ExtractMostSignificantBits(); do diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs index 3188dabe53b76..a1e68fcb044c3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs @@ -8,7 +8,8 @@ namespace System.Buffers { /// /// Provides an immutable, read-only set of values optimized for efficient searching. - /// Instances are created by or . + /// Instances are created by , , or + /// . /// /// The type of the values to search for. /// @@ -38,6 +39,9 @@ private protected SearchValues() { } internal virtual int LastIndexOfAny(ReadOnlySpan span) => throw new UnreachableException(); internal virtual int LastIndexOfAnyExcept(ReadOnlySpan span) => throw new UnreachableException(); + // This is only implemented and used by SearchValues. + internal virtual int IndexOfAnyMultiString(ReadOnlySpan span) => throw new UnreachableException(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int IndexOfAny(ReadOnlySpan span, SearchValues values) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index b7fee2b7b1245..ea5dc2915b99e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -9,8 +9,6 @@ using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; -#pragma warning disable 8500 // address of managed types - namespace System.Buffers { /// @@ -165,6 +163,22 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), return new ProbabilisticCharSearchValues(probabilisticValues); } + /// + /// Creates an optimized representation of used for efficient searching. + /// Only or may be used. + /// + /// The set of values. + /// Specifies whether to use or search semantics. + public static SearchValues Create(ReadOnlySpan values, StringComparison comparisonType) + { + if (comparisonType is not (StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException(SR.Argument_SearchValues_UnsupportedStringComparison, nameof(comparisonType)); + } + + return StringSearchValues.Create(values, ignoreCase: comparisonType == StringComparison.OrdinalIgnoreCase); + } + private static bool TryGetSingleRange(ReadOnlySpan values, out T minInclusive, out T maxInclusive) where T : struct, INumber, IMinMaxValue { @@ -209,12 +223,12 @@ internal interface IRuntimeConst static abstract bool Value { get; } } - private readonly struct TrueConst : IRuntimeConst + internal readonly struct TrueConst : IRuntimeConst { public static bool Value => true; } - private readonly struct FalseConst : IRuntimeConst + internal readonly struct FalseConst : IRuntimeConst { public static bool Value => false; } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs new file mode 100644 index 0000000000000..8a51d58106f54 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -0,0 +1,546 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +using static System.Buffers.StringSearchValuesHelper; +using static System.Buffers.TeddyHelper; + +namespace System.Buffers +{ + internal abstract class AsciiStringSearchValuesTeddyBase : StringSearchValuesRabinKarp + where TBucketized : struct, SearchValues.IRuntimeConst + where TStartCaseSensitivity : struct, ICaseSensitivity + where TCaseSensitivity : struct, ICaseSensitivity + { + private const int MatchStartOffsetN2 = 1; + private const int MatchStartOffsetN3 = 2; + private const int CharsPerIterationVector128 = 16; + private const int CharsPerIterationAvx2 = 32; + private const int CharsPerIterationAvx512 = 64; + + private readonly EightPackedReferences _buckets; + + private readonly Vector512 + _n0Low, _n0High, + _n1Low, _n1High, + _n2Low, _n2High; + + protected AsciiStringSearchValuesTeddyBase(ReadOnlySpan values, HashSet uniqueValues, int n) : base(values, uniqueValues) + { + if (TBucketized.Value) + { + throw new UnreachableException(); + } + + _buckets = new EightPackedReferences(MemoryMarshal.CreateReadOnlySpan( + ref Unsafe.As(ref MemoryMarshal.GetReference(values)), + values.Length)); + + (_n0Low, _n0High) = TeddyBucketizer.GenerateNonBucketizedFingerprint(values, offset: 0); + (_n1Low, _n1High) = TeddyBucketizer.GenerateNonBucketizedFingerprint(values, offset: 1); + + if (n == 3) + { + (_n2Low, _n2High) = TeddyBucketizer.GenerateNonBucketizedFingerprint(values, offset: 2); + } + } + + protected AsciiStringSearchValuesTeddyBase(string[][] buckets, ReadOnlySpan values, HashSet uniqueValues, int n) : base(values, uniqueValues) + { + if (!TBucketized.Value) + { + throw new UnreachableException(); + } + + _buckets = new EightPackedReferences(buckets); + + (_n0Low, _n0High) = TeddyBucketizer.GenerateBucketizedFingerprint(buckets, offset: 0); + (_n1Low, _n1High) = TeddyBucketizer.GenerateBucketizedFingerprint(buckets, offset: 1); + + if (n == 3) + { + (_n2Low, _n2High) = TeddyBucketizer.GenerateBucketizedFingerprint(buckets, offset: 2); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + protected int IndexOfAnyN2(ReadOnlySpan span) + { + // The behavior of the rest of the function remains the same if Avx2 or Avx512BW aren't supported +#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough + if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported && span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN2) + { + return IndexOfAnyN2Avx512(span); + } + + if (Avx2.IsSupported && span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN2) + { + return IndexOfAnyN2Avx2(span); + } +#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough + + return IndexOfAnyN2Vector128(span); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + protected int IndexOfAnyN3(ReadOnlySpan span) + { + // The behavior of the rest of the function remains the same if Avx2 or Avx512BW aren't supported +#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough + if (Vector512.IsHardwareAccelerated && Avx512BW.IsSupported && span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN3) + { + return IndexOfAnyN3Avx512(span); + } + + if (Avx2.IsSupported && span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN3) + { + return IndexOfAnyN3Avx2(span); + } +#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough + + return IndexOfAnyN3Vector128(span); + } + + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private int IndexOfAnyN2Vector128(ReadOnlySpan span) + { + if (span.Length < CharsPerIterationVector128 + MatchStartOffsetN2) + { + return ShortInputFallback(span); + } + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationVector128); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN2); + + Vector128 n0Low = _n0Low._lower._lower, n0High = _n0High._lower._lower; + Vector128 n1Low = _n1Low._lower._lower, n1High = _n1High._lower._lower; + Vector128 prev0 = Vector128.AllBitsSet; + + Loop: + Vector128 input = TStartCaseSensitivity.TransformInput(LoadAndPack16AsciiChars(ref searchSpace)); + + (Vector128 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); + + if (result != Vector128.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationVector128); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationVector128))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 no longer points to the characters just before the current input, so we must reset it. + prev0 = Vector128.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN2, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [CompExactlyDependsOn(typeof(Avx2))] + private int IndexOfAnyN2Avx2(ReadOnlySpan span) + { + Debug.Assert(span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN2); + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationAvx2); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN2); + + Vector256 n0Low = _n0Low._lower, n0High = _n0High._lower; + Vector256 n1Low = _n1Low._lower, n1High = _n1High._lower; + Vector256 prev0 = Vector256.AllBitsSet; + + Loop: + Vector256 input = TStartCaseSensitivity.TransformInput(LoadAndPack32AsciiChars(ref searchSpace)); + + (Vector256 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); + + if (result != Vector256.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationAvx2); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationAvx2))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 no longer points to the characters just before the current input, so we must reset it. + prev0 = Vector256.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN2, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [CompExactlyDependsOn(typeof(Avx512BW))] + private int IndexOfAnyN2Avx512(ReadOnlySpan span) + { + Debug.Assert(span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN2); + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationAvx512); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN2); + + Vector512 n0Low = _n0Low, n0High = _n0High; + Vector512 n1Low = _n1Low, n1High = _n1High; + Vector512 prev0 = Vector512.AllBitsSet; + + Loop: + Vector512 input = TStartCaseSensitivity.TransformInput(LoadAndPack64AsciiChars(ref searchSpace)); + + (Vector512 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); + + if (result != Vector512.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationAvx512); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationAvx512))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 no longer points to the characters just before the current input, so we must reset it. + prev0 = Vector512.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN2, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private int IndexOfAnyN3Vector128(ReadOnlySpan span) + { + if (span.Length < CharsPerIterationVector128 + MatchStartOffsetN3) + { + return ShortInputFallback(span); + } + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationVector128); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN3); + + Vector128 n0Low = _n0Low._lower._lower, n0High = _n0High._lower._lower; + Vector128 n1Low = _n1Low._lower._lower, n1High = _n1High._lower._lower; + Vector128 n2Low = _n2Low._lower._lower, n2High = _n2High._lower._lower; + Vector128 prev0 = Vector128.AllBitsSet; + Vector128 prev1 = Vector128.AllBitsSet; + + Loop: + Vector128 input = TStartCaseSensitivity.TransformInput(LoadAndPack16AsciiChars(ref searchSpace)); + + (Vector128 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); + + if (result != Vector128.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationVector128); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationVector128))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 and prev1 no longer point to the characters just before the current input, so we must reset them. + prev0 = Vector128.AllBitsSet; + prev1 = Vector128.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN3, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [CompExactlyDependsOn(typeof(Avx2))] + private int IndexOfAnyN3Avx2(ReadOnlySpan span) + { + Debug.Assert(span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN3); + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationAvx2); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN3); + + Vector256 n0Low = _n0Low._lower, n0High = _n0High._lower; + Vector256 n1Low = _n1Low._lower, n1High = _n1High._lower; + Vector256 n2Low = _n2Low._lower, n2High = _n2High._lower; + Vector256 prev0 = Vector256.AllBitsSet; + Vector256 prev1 = Vector256.AllBitsSet; + + Loop: + Vector256 input = TStartCaseSensitivity.TransformInput(LoadAndPack32AsciiChars(ref searchSpace)); + + (Vector256 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); + + if (result != Vector256.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationAvx2); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationAvx2))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 and prev1 no longer point to the characters just before the current input, so we must reset them. + prev0 = Vector256.AllBitsSet; + prev1 = Vector256.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN3, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [CompExactlyDependsOn(typeof(Avx512BW))] + private int IndexOfAnyN3Avx512(ReadOnlySpan span) + { + Debug.Assert(span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN3); + + ref char searchSpace = ref MemoryMarshal.GetReference(span); + ref char lastSearchSpaceStart = ref Unsafe.Add(ref searchSpace, span.Length - CharsPerIterationAvx512); + + searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN3); + + Vector512 n0Low = _n0Low, n0High = _n0High; + Vector512 n1Low = _n1Low, n1High = _n1High; + Vector512 n2Low = _n2Low, n2High = _n2High; + Vector512 prev0 = Vector512.AllBitsSet; + Vector512 prev1 = Vector512.AllBitsSet; + + Loop: + Vector512 input = TStartCaseSensitivity.TransformInput(LoadAndPack64AsciiChars(ref searchSpace)); + + (Vector512 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); + + if (result != Vector512.Zero) + { + goto CandidateFound; + } + + ContinueLoop: + searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationAvx512); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpaceStart, CharsPerIterationAvx512))) + { + return -1; + } + + // We're switching which characters we will process in the next iteration. + // prev0 and prev1 no longer point to the characters just before the current input, so we must reset them. + prev0 = Vector512.AllBitsSet; + prev1 = Vector512.AllBitsSet; + searchSpace = ref lastSearchSpaceStart; + } + goto Loop; + + CandidateFound: + if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN3, out int offset)) + { + return offset; + } + goto ContinueLoop; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector128 result, int matchStartOffset, out int offsetFromStart) + { + uint resultMask = (~Vector128.Equals(result, Vector128.Zero)).ExtractMostSignificantBits(); + + do + { + int matchOffset = BitOperations.TrailingZeroCount(resultMask); + + ref char matchRef = ref Unsafe.Add(ref searchSpace, matchOffset - matchStartOffset); + offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); + int lengthRemaining = span.Length - offsetFromStart; + + uint candidateMask = result.GetElementUnsafe(matchOffset); + + do + { + int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); + + object bucket = _buckets[candidateOffset]; + + if (TBucketized.Value + ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) + : StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket))) + { + return true; + } + + candidateMask = BitOperations.ResetLowestSetBit(candidateMask); + } + while (candidateMask != 0); + + resultMask = BitOperations.ResetLowestSetBit(resultMask); + } + while (resultMask != 0); + + offsetFromStart = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector256 result, int matchStartOffset, out int offsetFromStart) + { + uint resultMask = (~Vector256.Equals(result, Vector256.Zero)).ExtractMostSignificantBits(); + + do + { + int matchOffset = BitOperations.TrailingZeroCount(resultMask); + + ref char matchRef = ref Unsafe.Add(ref searchSpace, matchOffset - matchStartOffset); + offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); + int lengthRemaining = span.Length - offsetFromStart; + + uint candidateMask = result.GetElementUnsafe(matchOffset); + + do + { + int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); + + object bucket = _buckets[candidateOffset]; + + if (TBucketized.Value + ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) + : StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket))) + { + return true; + } + + candidateMask = BitOperations.ResetLowestSetBit(candidateMask); + } + while (candidateMask != 0); + + resultMask = BitOperations.ResetLowestSetBit(resultMask); + } + while (resultMask != 0); + + offsetFromStart = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector512 result, int matchStartOffset, out int offsetFromStart) + { + ulong resultMask = (~Vector512.Equals(result, Vector512.Zero)).ExtractMostSignificantBits(); + + do + { + int matchOffset = BitOperations.TrailingZeroCount(resultMask); + + ref char matchRef = ref Unsafe.Add(ref searchSpace, matchOffset - matchStartOffset); + offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); + int lengthRemaining = span.Length - offsetFromStart; + + uint candidateMask = result.GetElementUnsafe(matchOffset); + + do + { + int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); + + object bucket = _buckets[candidateOffset]; + + if (TBucketized.Value + ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) + : StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket))) + { + return true; + } + + candidateMask = BitOperations.ResetLowestSetBit(candidateMask); + } + while (candidateMask != 0); + + resultMask = BitOperations.ResetLowestSetBit(resultMask); + } + while (resultMask != 0); + + offsetFromStart = 0; + return false; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs new file mode 100644 index 0000000000000..be7a7140134b2 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class AsciiStringSearchValuesTeddyBucketizedN2 : AsciiStringSearchValuesTeddyBase + where TStartCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + public AsciiStringSearchValuesTeddyBucketizedN2(string[][] buckets, ReadOnlySpan values, HashSet uniqueValues) + : base(buckets, values, uniqueValues, n: 2) + { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + IndexOfAnyN2(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs new file mode 100644 index 0000000000000..2698d02245aa0 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class AsciiStringSearchValuesTeddyBucketizedN3 : AsciiStringSearchValuesTeddyBase + where TStartCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + public AsciiStringSearchValuesTeddyBucketizedN3(string[][] buckets, ReadOnlySpan values, HashSet uniqueValues) + : base(buckets, values, uniqueValues, n: 3) + { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + IndexOfAnyN3(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs new file mode 100644 index 0000000000000..1dbfa78efd88d --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class AsciiStringSearchValuesTeddyNonBucketizedN2 : AsciiStringSearchValuesTeddyBase + where TStartCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + public AsciiStringSearchValuesTeddyNonBucketizedN2(ReadOnlySpan values, HashSet uniqueValues) + : base(values, uniqueValues, n: 2) + { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + IndexOfAnyN2(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs new file mode 100644 index 0000000000000..63efaa814514c --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class AsciiStringSearchValuesTeddyNonBucketizedN3 : AsciiStringSearchValuesTeddyBase + where TStartCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + public AsciiStringSearchValuesTeddyNonBucketizedN3(ReadOnlySpan values, HashSet uniqueValues) + : base(values, uniqueValues, n: 3) + { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + IndexOfAnyN3(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs new file mode 100644 index 0000000000000..bcc7c84bc48c8 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -0,0 +1,548 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace System.Buffers +{ + /// + /// An implementation of the Aho-Corasick algorithm we use as a fallback when we can't use Teddy + /// (either due to missing hardware intrinsics, or due to characteristics of the of values used). + /// https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm + /// + internal readonly struct AhoCorasick + { + private readonly Node[] _nodes; + private readonly Vector256 _startingCharsAsciiBitmap; + private readonly int _maxValueLength; // Only used by the NLS fallback + + public AhoCorasick(Node[] nodes, Vector256 startingAsciiBitmap, int maxValueLength) + { + _nodes = nodes; + _startingCharsAsciiBitmap = startingAsciiBitmap; + _maxValueLength = maxValueLength; + } + + public readonly bool ShouldUseAsciiFastScan + { + get + { + Vector256 bitmap = _startingCharsAsciiBitmap; + + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && bitmap != default) + { + // If there are a lot of starting characters such that we often find one early, + // the ASCII fast scan may end up performing worse than checking one character at a time. + // Avoid using this optimization if the combined frequency of starting chars is too high. + + // Combined frequency of + // - All digits is ~ 5 + // - All lowercase letters is ~ 57.2 + // - All uppercase letters is ~ 7.4 + const float MaxCombinedFrequency = 50f; + + float frequency = 0; + + for (int i = 0; i < 128; i++) + { + if (IndexOfAnyAsciiSearcher.BitmapContains(ref bitmap, (char)i)) + { + frequency += CharacterFrequencyHelper.AsciiFrequency[i]; + } + } + + return frequency < MaxCombinedFrequency; + } + + return false; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly int IndexOfAny(ReadOnlySpan span) + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TFastScanVariant : struct, IFastScan + { + if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) + { + return GlobalizationMode.UseNls + ? IndexOfAnyCaseInsensitiveUnicodeNls(span) + : IndexOfAnyCaseInsensitiveUnicodeIcuOrInvariant(span); + } + + return IndexOfAnyCore(span); + } + + private readonly int IndexOfAnyCore(ReadOnlySpan span) + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TFastScanVariant : struct, IFastScan + { + if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) + { + throw new UnreachableException(); + } + + ref Node nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); + int nodeIndex = 0; + int result = -1; + int i = 0; + + FastScan: + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && typeof(TFastScanVariant) == typeof(IndexOfAnyAsciiFastScan)) + { + int remainingLength = span.Length - i; + + if (remainingLength >= Vector128.Count) + { + // If '\0' is one of the starting chars and we're running on Ssse3 hardware, this may return false-positives. + // False-positives here are okay, we'll just rule them out below. While we could flow the Ssse3AndWasmHandleZeroInNeedle + // generic through, we expect such values to be rare enough that introducing more code is not worth it. + int offset = IndexOfAnyAsciiSearcher.IndexOfAnyVectorized( + ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), i)), + remainingLength, + ref Unsafe.AsRef(_startingCharsAsciiBitmap)); + + if (offset < 0) + { + goto Return; + } + + i += offset; + goto LoopWithoutRangeCheck; + } + } + + Loop: + if ((uint)i >= (uint)span.Length) + { + goto Return; + } + + LoopWithoutRangeCheck: + Debug.Assert(i < span.Length); + char c = TCaseSensitivity.TransformInput(Unsafe.Add(ref MemoryMarshal.GetReference(span), i)); + + while (true) + { + Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); + ref Node node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); + + if (node.TryGetChild(c, out int childIndex)) + { + nodeIndex = childIndex; + + int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; + if (matchLength != 0) + { + result = i + 1 - matchLength; + } + + i++; + goto Loop; + } + + if (nodeIndex == 0) + { + if (result >= 0) + { + goto Return; + } + + i++; + goto FastScan; + } + + nodeIndex = node.SuffixLink; + + if (nodeIndex < 0) + { + Debug.Assert(nodeIndex == -1); + Debug.Assert(result >= 0); + goto Return; + } + } + + Return: + return result; + } + + // Mostly a copy of IndexOfAnyCore, but we may read two characters at a time in the case of surrogate pairs. + private readonly int IndexOfAnyCaseInsensitiveUnicodeIcuOrInvariant(ReadOnlySpan span) + where TFastScanVariant : struct, IFastScan + { + Debug.Assert(!GlobalizationMode.UseNls); + + const char LowSurrogateNotSet = '\0'; + + ref Node nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); + int nodeIndex = 0; + int result = -1; + int i = 0; + char lowSurrogateUpper = LowSurrogateNotSet; + + FastScan: + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && typeof(TFastScanVariant) == typeof(IndexOfAnyAsciiFastScan)) + { + if (lowSurrogateUpper != LowSurrogateNotSet) + { + goto LoopWithoutRangeCheck; + } + + int remainingLength = span.Length - i; + + if (remainingLength >= Vector128.Count) + { + int offset = IndexOfAnyAsciiSearcher.IndexOfAnyVectorized( + ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), i)), + remainingLength, + ref Unsafe.AsRef(_startingCharsAsciiBitmap)); + + if (offset < 0) + { + goto Return; + } + + i += offset; + goto LoopWithoutRangeCheck; + } + } + + Loop: + if ((uint)i >= (uint)span.Length) + { + goto Return; + } + + LoopWithoutRangeCheck: + Debug.Assert(i < span.Length); + char c; + if (lowSurrogateUpper != LowSurrogateNotSet) + { + c = lowSurrogateUpper; + lowSurrogateUpper = LowSurrogateNotSet; + } + else + { + c = Unsafe.Add(ref MemoryMarshal.GetReference(span), i); + char lowSurrogate; + + if (char.IsHighSurrogate(c) && + (uint)(i + 1) < (uint)span.Length && + char.IsLowSurrogate(lowSurrogate = Unsafe.Add(ref MemoryMarshal.GetReference(span), i + 1))) + { + SurrogateCasing.ToUpper(c, lowSurrogate, out c, out lowSurrogateUpper); + Debug.Assert(lowSurrogateUpper != LowSurrogateNotSet); + } + else + { + c = GlobalizationMode.Invariant + ? InvariantModeCasing.ToUpper(c) + : OrdinalCasing.ToUpper(c); + } + +#if DEBUG + // This logic must match Ordinal.ToUpperOrdinal exactly. + Span destination = new char[2]; // Avoid stackalloc in a loop + Ordinal.ToUpperOrdinal(span.Slice(i, i + 1 == span.Length ? 1 : 2), destination); + Debug.Assert(c == destination[0]); + Debug.Assert(lowSurrogateUpper == LowSurrogateNotSet || lowSurrogateUpper == destination[1]); +#endif + } + + while (true) + { + Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); + ref Node node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); + + if (node.TryGetChild(c, out int childIndex)) + { + nodeIndex = childIndex; + + int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; + if (matchLength != 0) + { + result = i + 1 - matchLength; + } + + i++; + goto Loop; + } + + if (nodeIndex == 0) + { + if (result >= 0) + { + goto Return; + } + + i++; + goto FastScan; + } + + nodeIndex = node.SuffixLink; + + if (nodeIndex < 0) + { + Debug.Assert(nodeIndex == -1); + Debug.Assert(result >= 0); + goto Return; + } + } + + Return: + return result; + } + + private readonly int IndexOfAnyCaseInsensitiveUnicodeNls(ReadOnlySpan span) + where TFastScanVariant : struct, IFastScan + { + Debug.Assert(GlobalizationMode.UseNls); + + if (span.IsEmpty) + { + return -1; + } + + // If the input is large, we avoid uppercasing all of it upfront. + // We may find a match at position 0, so we want to behave closer to O(match offset) than O(input length). +#if DEBUG + // Make it easier to test with shorter inputs + const int StackallocThreshold = 32; +#else + // This limit isn't just about how much we allocate on the stack, but also how we chunk the input span. + // A larger value would improve throughput for rare matches, while a lower number reduces the overhead + // when matches are found close to the start. + const int StackallocThreshold = 64; +#endif + + int minBufferSize = (int)Math.Clamp(_maxValueLength * 4L, StackallocThreshold, string.MaxLength + 1); + + char[]? pooledArray = null; + Span buffer = minBufferSize <= StackallocThreshold + ? stackalloc char[StackallocThreshold] + : (pooledArray = ArrayPool.Shared.Rent(minBufferSize)); + + int leftoverFromPreviousIteration = 0; + int offsetFromStart = 0; + int result; + + while (true) + { + Span newSpaceAvailable = buffer.Slice(leftoverFromPreviousIteration); + int toConvert = Math.Min(span.Length, newSpaceAvailable.Length); + + int charsWritten = Ordinal.ToUpperOrdinal(span.Slice(0, toConvert), newSpaceAvailable); + Debug.Assert(charsWritten == toConvert); + span = span.Slice(toConvert); + + Span upperCaseBuffer = buffer.Slice(0, leftoverFromPreviousIteration + toConvert); + + // CaseSensitive instead of CaseInsensitiveUnicode as we've already done the case conversion. + result = IndexOfAnyCore(upperCaseBuffer); + + // Even if we found a result, it is possible that an earlier match exists if we ran out of upperCaseBuffer. + // If that is the case, we will find the correct result in the next loop iteration. + if (result >= 0 && (span.IsEmpty || result <= buffer.Length - _maxValueLength)) + { + result += offsetFromStart; + break; + } + + if (span.IsEmpty) + { + result = -1; + break; + } + + leftoverFromPreviousIteration = _maxValueLength - 1; + buffer.Slice(buffer.Length - leftoverFromPreviousIteration).CopyTo(buffer); + offsetFromStart += buffer.Length - leftoverFromPreviousIteration; + } + + if (pooledArray is not null) + { + ArrayPool.Shared.Return(pooledArray); + } + + return result; + } + + public interface IFastScan { } + + public readonly struct IndexOfAnyAsciiFastScan : IFastScan { } + + public readonly struct NoFastScan : IFastScan { } + + [DebuggerDisplay("MatchLength={MatchLength} SuffixLink={SuffixLink} ChildrenCount={(_children?.Count ?? 0) + (_firstChildChar < 0 ? 0 : 1)}")] + public struct Node + { + private static object EmptyChildrenSentinel => Array.Empty(); + + public int SuffixLink; + public int MatchLength; + + private int _firstChildChar; + private int _firstChildIndex; + private object _children; // Either Dictionary or int[] + + public Node() + { + _firstChildChar = -1; + _children = EmptyChildrenSentinel; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TryGetChild(char c, out int index) + { + if (_firstChildChar == c) + { + index = _firstChildIndex; + return true; + } + + object children = _children; + Debug.Assert(children is int[] || children is Dictionary); + + if (children.GetType() == typeof(int[])) + { + int[] table = Unsafe.As(children); + if (c < (uint)table.Length) + { + index = table[c]; + if (index >= 0) + { + return true; + } + } + } + else + { + return Unsafe.As>(children).TryGetValue(c, out index); + } + + index = 0; + return false; + } + + public void AddChild(char c, int index) + { + if (_firstChildChar < 0) + { + _firstChildChar = c; + _firstChildIndex = index; + } + else + { + if (ReferenceEquals(_children, EmptyChildrenSentinel)) + { + _children = new Dictionary(); + } + + ((Dictionary)_children).Add(c, index); + } + } + + public readonly void AddChildrenToQueue(Queue<(char Char, int Index)> queue) + { + if (_firstChildChar >= 0) + { + queue.Enqueue(((char)_firstChildChar, _firstChildIndex)); + + if (_children is Dictionary children) + { + foreach ((char childChar, int childIndex) in children) + { + queue.Enqueue((childChar, childIndex)); + } + } + } + } + + public void OptimizeChildren() + { + if (_children is Dictionary children) + { + children.Add((char)_firstChildChar, _firstChildIndex); + + float frequency = -2; + + foreach ((char childChar, int childIndex) in children) + { + float newFrequency = char.IsAscii(childChar) ? CharacterFrequencyHelper.AsciiFrequency[childChar] : -1; + + if (newFrequency > frequency) + { + frequency = newFrequency; + _firstChildChar = childChar; + _firstChildIndex = childIndex; + } + } + + children.Remove((char)_firstChildChar); + + if (TryCreateJumpTable(children, out int[]? table)) + { + _children = table; + } + } + + static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true)] out int[]? table) + { + // Sacrifice some memory usage in exchange for faster lookup performance + const int AcceptableSizeMultiplier = 2; + + Debug.Assert(children.Count > 0); + + int maxValue = -1; + + foreach ((char childChar, _) in children) + { + maxValue = Math.Max(maxValue, childChar); + } + + int tableSize = TableSizeEstimate(maxValue); + int dictionarySize = DictionarySizeEstimate(children.Count); + + if (tableSize > dictionarySize * AcceptableSizeMultiplier) + { + // We would have a lot of empty entries. Avoid wasting too much memory. + table = null; + return false; + } + + table = new int[maxValue + 1]; + Array.Fill(table, -1); + + foreach ((char childChar, int childIndex) in children) + { + table[childChar] = childIndex; + } + + return true; + + static int TableSizeEstimate(int maxValue) + { + return 32 + (maxValue * 4); + } + + static int DictionarySizeEstimate(int childCount) + { + return childCount switch + { + < 4 => 192, + < 8 => 272, + < 12 => 352, + _ => childCount * 25 + }; + } + } + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs new file mode 100644 index 0000000000000..951ee9bc5417e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs @@ -0,0 +1,191 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.Intrinsics; +using System.Text; + +namespace System.Buffers +{ + /// + /// Separated out of to allow us to defer some computation costs in case we decide not to build the full thing. + /// + internal ref struct AhoCorasickBuilder + { + private readonly ReadOnlySpan _values; + private readonly bool _ignoreCase; + private ValueListBuilder _nodes; + private ValueListBuilder _parents; + private Vector256 _startingCharsAsciiBitmap; + private int _maxValueLength; // Only used by the NLS fallback + + public AhoCorasickBuilder(ReadOnlySpan values, bool ignoreCase, ref HashSet? unreachableValues) + { + Debug.Assert(!values.IsEmpty); + Debug.Assert(!string.IsNullOrEmpty(values[0])); + +#if DEBUG + // The input should have been sorted by length + for (int i = 1; i < values.Length; i++) + { + Debug.Assert(values[i - 1].Length <= values[i].Length); + } +#endif + + _values = values; + _ignoreCase = ignoreCase; + BuildTrie(ref unreachableValues); + } + + public AhoCorasick Build() + { + AddSuffixLinks(); + + Debug.Assert(_nodes[0].MatchLength == 0, "The root node shouldn't have a match."); + + for (int i = 0; i < _nodes.Length; i++) + { + _nodes[i].OptimizeChildren(); + } + + if (IndexOfAnyAsciiSearcher.IsVectorizationSupported) + { + GenerateStartingAsciiCharsBitmap(); + } + + return new AhoCorasick(_nodes.AsSpan().ToArray(), _startingCharsAsciiBitmap, _maxValueLength); + } + + public void Dispose() + { + _nodes.Dispose(); + _parents.Dispose(); + } + + private void BuildTrie(ref HashSet? unreachableValues) + { + _nodes.Append(new AhoCorasick.Node()); + _parents.Append(0); + + foreach (string value in _values) + { + int nodeIndex = 0; + ref AhoCorasick.Node node = ref _nodes[nodeIndex]; + + for (int i = 0; i < value.Length; i++) + { + char c = value[i]; + + if (!node.TryGetChild(c, out int childIndex)) + { + childIndex = _nodes.Length; + node.AddChild(c, childIndex); + _nodes.Append(new AhoCorasick.Node()); + _parents.Append(nodeIndex); + } + + node = ref _nodes[childIndex]; + nodeIndex = childIndex; + + if (node.MatchLength != 0) + { + // A previous value is an exact prefix of this one. + // We're looking for the index of the first match, not necessarily the longest one, we can skip this value. + // We've already normalized the values, so we can do ordinal comparisons here. + unreachableValues ??= new HashSet(StringComparer.Ordinal); + unreachableValues.Add(value); + break; + } + + if (i == value.Length - 1) + { + node.MatchLength = value.Length; + _maxValueLength = Math.Max(_maxValueLength, value.Length); + break; + } + } + } + } + + private void AddSuffixLinks() + { + var queue = new Queue<(char Char, int Index)>(); + queue.Enqueue(((char)0, 0)); + + while (queue.TryDequeue(out (char Char, int Index) trieNode)) + { + ref AhoCorasick.Node node = ref _nodes[trieNode.Index]; + int parent = _parents[trieNode.Index]; + int suffixLink = _nodes[parent].SuffixLink; + + if (parent != 0) + { + while (suffixLink >= 0) + { + ref AhoCorasick.Node suffixNode = ref _nodes[suffixLink]; + + if (suffixNode.TryGetChild(trieNode.Char, out int childSuffixLink)) + { + suffixLink = childSuffixLink; + break; + } + + if (suffixLink == 0) + { + break; + } + + suffixLink = suffixNode.SuffixLink; + } + } + + if (node.MatchLength != 0) + { + node.SuffixLink = -1; + + // If a node is a match, there's no need to assign suffix links to its children. + // If a child does not match, such that we would look at its suffix link, we already saw an earlier match node. + } + else + { + node.SuffixLink = suffixLink; + + if (suffixLink >= 0) + { + node.MatchLength = _nodes[suffixLink].MatchLength; + } + + node.AddChildrenToQueue(queue); + } + } + } + + private void GenerateStartingAsciiCharsBitmap() + { + scoped ValueListBuilder startingChars = new ValueListBuilder(stackalloc char[128]); + + foreach (string value in _values) + { + char c = value[0]; + + if (_ignoreCase) + { + startingChars.Append(char.ToLowerInvariant(c)); + startingChars.Append(char.ToUpperInvariant(c)); + } + else + { + startingChars.Append(c); + } + } + + if (Ascii.IsValid(startingChars.AsSpan())) + { + IndexOfAnyAsciiSearcher.ComputeBitmap(startingChars.AsSpan(), out _startingCharsAsciiBitmap, out _); + } + + startingChars.Dispose(); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/CharacterFrequencyHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/CharacterFrequencyHelper.cs new file mode 100644 index 0000000000000..156c925c755f9 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/CharacterFrequencyHelper.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; + +namespace System.Buffers +{ + internal static class CharacterFrequencyHelper + { + // Same as RegexPrefixAnalyzer.Frequency. + // https://github.com/dotnet/runtime/blob/a355d5f7db162714ee19533ca55074aa2cbd8a8c/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs#L956C43-L956C53 + public static ReadOnlySpan AsciiFrequency => new float[] + { + 0.000f /* '\x00' */, 0.000f /* '\x01' */, 0.000f /* '\x02' */, 0.000f /* '\x03' */, 0.000f /* '\x04' */, 0.000f /* '\x05' */, 0.000f /* '\x06' */, 0.000f /* '\x07' */, + 0.000f /* '\x08' */, 0.001f /* '\x09' */, 0.000f /* '\x0A' */, 0.000f /* '\x0B' */, 0.000f /* '\x0C' */, 0.000f /* '\x0D' */, 0.000f /* '\x0E' */, 0.000f /* '\x0F' */, + 0.000f /* '\x10' */, 0.000f /* '\x11' */, 0.000f /* '\x12' */, 0.000f /* '\x13' */, 0.003f /* '\x14' */, 0.000f /* '\x15' */, 0.000f /* '\x16' */, 0.000f /* '\x17' */, + 0.000f /* '\x18' */, 0.004f /* '\x19' */, 0.000f /* '\x1A' */, 0.000f /* '\x1B' */, 0.006f /* '\x1C' */, 0.006f /* '\x1D' */, 0.000f /* '\x1E' */, 0.000f /* '\x1F' */, + 8.952f /* ' ' */, 0.065f /* ' !' */, 0.420f /* ' "' */, 0.010f /* ' #' */, 0.011f /* ' $' */, 0.005f /* ' %' */, 0.070f /* ' &' */, 0.050f /* ' '' */, + 3.911f /* ' (' */, 3.910f /* ' )' */, 0.356f /* ' *' */, 2.775f /* ' +' */, 1.411f /* ' ,' */, 0.173f /* ' -' */, 2.054f /* ' .' */, 0.677f /* ' /' */, + 1.199f /* ' 0' */, 0.870f /* ' 1' */, 0.729f /* ' 2' */, 0.491f /* ' 3' */, 0.335f /* ' 4' */, 0.269f /* ' 5' */, 0.435f /* ' 6' */, 0.240f /* ' 7' */, + 0.234f /* ' 8' */, 0.196f /* ' 9' */, 0.144f /* ' :' */, 0.983f /* ' ;' */, 0.357f /* ' <' */, 0.661f /* ' =' */, 0.371f /* ' >' */, 0.088f /* ' ?' */, + 0.007f /* ' @' */, 0.763f /* ' A' */, 0.229f /* ' B' */, 0.551f /* ' C' */, 0.306f /* ' D' */, 0.449f /* ' E' */, 0.337f /* ' F' */, 0.162f /* ' G' */, + 0.131f /* ' H' */, 0.489f /* ' I' */, 0.031f /* ' J' */, 0.035f /* ' K' */, 0.301f /* ' L' */, 0.205f /* ' M' */, 0.253f /* ' N' */, 0.228f /* ' O' */, + 0.288f /* ' P' */, 0.034f /* ' Q' */, 0.380f /* ' R' */, 0.730f /* ' S' */, 0.675f /* ' T' */, 0.265f /* ' U' */, 0.309f /* ' V' */, 0.137f /* ' W' */, + 0.084f /* ' X' */, 0.023f /* ' Y' */, 0.023f /* ' Z' */, 0.591f /* ' [' */, 0.085f /* ' \' */, 0.590f /* ' ]' */, 0.013f /* ' ^' */, 0.797f /* ' _' */, + 0.001f /* ' `' */, 4.596f /* ' a' */, 1.296f /* ' b' */, 2.081f /* ' c' */, 2.005f /* ' d' */, 6.903f /* ' e' */, 1.494f /* ' f' */, 1.019f /* ' g' */, + 1.024f /* ' h' */, 3.750f /* ' i' */, 0.286f /* ' j' */, 0.439f /* ' k' */, 2.913f /* ' l' */, 1.459f /* ' m' */, 3.908f /* ' n' */, 3.230f /* ' o' */, + 1.444f /* ' p' */, 0.231f /* ' q' */, 4.220f /* ' r' */, 3.924f /* ' s' */, 5.312f /* ' t' */, 2.112f /* ' u' */, 0.737f /* ' v' */, 0.573f /* ' w' */, + 0.992f /* ' x' */, 1.067f /* ' y' */, 0.181f /* ' z' */, 0.391f /* ' {' */, 0.056f /* ' |' */, 0.391f /* ' }' */, 0.002f /* ' ~' */, 0.000f /* '\x7F' */, + }; + + public static void GetSingleStringMultiCharacterOffsets(string value, bool ignoreCase, out int ch2Offset, out int ch3Offset) + { + Debug.Assert(value.Length > 1); + Debug.Assert(!ignoreCase || char.IsAscii(value[0])); + + ch2Offset = IndexOfAsciiCharWithLowestFrequency(value, ignoreCase); + ch3Offset = 0; + + if (ch2Offset < 0) + { + // We have fewer than 2 ASCII chars in the value. + Debug.Assert(!ignoreCase); + + // We don't have a frequency table for non-ASCII characters, pick a random one. + ch2Offset = value.Length - 1; + } + + if (value.Length > 2) + { + ch3Offset = IndexOfAsciiCharWithLowestFrequency(value, ignoreCase, excludeIndex: ch2Offset); + + if (ch3Offset < 0) + { + // We have fewer than 3 ASCII chars in the value. + if (ignoreCase) + { + // We can still use N=2. + ch3Offset = 0; + } + else + { + // We don't have a frequency table for non-ASCII characters, pick a random one. + ch3Offset = value.Length - 1; + + if (ch2Offset == ch3Offset) + { + ch2Offset--; + } + } + } + } + + Debug.Assert(ch2Offset != 0); + Debug.Assert(ch2Offset != ch3Offset); + + if (ch3Offset > 0 && ch3Offset < ch2Offset) + { + (ch2Offset, ch3Offset) = (ch3Offset, ch2Offset); + } + } + + private static int IndexOfAsciiCharWithLowestFrequency(ReadOnlySpan span, bool ignoreCase, int excludeIndex = -1) + { + float minFrequency = float.MaxValue; + int minIndex = -1; + + // Exclude i = 0 as we've already decided to use the first character. + for (int i = 1; i < span.Length; i++) + { + if (i == excludeIndex) + { + continue; + } + + char c = span[i]; + + // We don't have a frequency table for non-ASCII characters, so they are ignored. + if (char.IsAscii(c)) + { + float frequency = AsciiFrequency[c]; + + if (ignoreCase) + { + // Include the alternative character that will also match. + frequency += AsciiFrequency[c ^ 0x20]; + } + + // Avoiding characters from the front of the value for the 2nd and 3rd character + // results in 18 % fewer false positive 3-char matches on "The Adventures of Sherlock Holmes". + if (i <= 2) + { + frequency *= 1.5f; + } + + if (frequency <= minFrequency) + { + minFrequency = frequency; + minIndex = i; + } + } + } + + return minIndex; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs new file mode 100644 index 0000000000000..16ca38d813375 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + [StructLayout(LayoutKind.Sequential)] + internal readonly struct EightPackedReferences + { + private readonly object? _ref0; + private readonly object? _ref1; + private readonly object? _ref2; + private readonly object? _ref3; + private readonly object? _ref4; + private readonly object? _ref5; + private readonly object? _ref6; + private readonly object? _ref7; + + public EightPackedReferences(ReadOnlySpan values) + { + Debug.Assert(values.Length <= 8, $"Got {values.Length} values"); + + for (int i = 0; i < values.Length; i++) + { + this[i] = values[i]; + } + } + + public object this[int index] + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + Debug.Assert(index is >= 0 and < 8, $"Should be [0, 7], was {index}"); + Debug.Assert(Unsafe.Add(ref Unsafe.AsRef(in _ref0), index) is not null); + + return Unsafe.Add(ref Unsafe.AsRef(in _ref0), index)!; + } + private set + { + Unsafe.Add(ref Unsafe.AsRef(in _ref0), index) = value; + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs new file mode 100644 index 0000000000000..3a707ac619812 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Buffers +{ + /// + /// An implementation of the Rabin-Karp algorithm we use as a fallback for + /// short inputs that we can't handle with Teddy. + /// https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm + /// + internal readonly struct RabinKarp + { + // The number of values we'll accept before falling back to Aho-Corasick. + // This also affects when Teddy may be used. + public const int MaxValues = 80; + + // This is a tradeoff between memory consumption and the number of false positives + // we have to rule out during the verification step. + private const nuint BucketCount = 64; + + // 18 = Vector128.Count + 2 (MatchStartOffset for N=3) + // The logic in this class is not safe from overflows, but we avoid any issues by + // only calling into it for inputs that are too short for Teddy to handle. + private const int MaxInputLength = 18 - 1; + + // We're using nuint as the rolling hash, so we can spread the hash over more bits on 64bit. + private static int HashShiftPerElement => IntPtr.Size == 8 ? 2 : 1; + + private readonly string[][] _buckets; + private readonly int _hashLength; + private readonly nuint _hashUpdateMultiplier; + + public RabinKarp(ReadOnlySpan values) + { + Debug.Assert(values.Length <= MaxValues); + + int minimumLength = int.MaxValue; + foreach (string value in values) + { + minimumLength = Math.Min(minimumLength, value.Length); + } + + Debug.Assert(minimumLength > 1); + + _hashLength = minimumLength; + _hashUpdateMultiplier = (nuint)1 << ((minimumLength - 1) * HashShiftPerElement); + + var bucketLists = new List?[BucketCount]; + + foreach (string value in values) + { + nuint hash = 0; + for (int i = 0; i < minimumLength; i++) + { + hash = (hash << HashShiftPerElement) + value[i]; + } + + nuint bucket = hash % BucketCount; + var bucketList = bucketLists[bucket] ??= new List(); + bucketList.Add(value); + } + + var buckets = new string[BucketCount][]; + for (int i = 0; i < bucketLists.Length; i++) + { + if (bucketLists[i] is List list) + { + buckets[i] = list.ToArray(); + } + } + + _buckets = buckets; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly int IndexOfAny(ReadOnlySpan span) + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity => + typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode) + ? IndexOfAnyCaseInsensitiveUnicode(span) + : IndexOfAnyCore(span); + + private readonly int IndexOfAnyCore(ReadOnlySpan span) + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) + { + throw new UnreachableException(); + } + + Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled short inputs."); + + ref char current = ref MemoryMarshal.GetReference(span); + + int hashLength = _hashLength; + + if (span.Length >= hashLength) + { + ref char end = ref Unsafe.Add(ref MemoryMarshal.GetReference(span), (uint)(span.Length - hashLength)); + + nuint hash = 0; + for (uint i = 0; i < hashLength; i++) + { + hash = (hash << HashShiftPerElement) + TCaseSensitivity.TransformInput(Unsafe.Add(ref current, i)); + } + + string[][] buckets = _buckets; + + while (true) + { + if (Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(buckets), hash % BucketCount) is string[] bucket) + { + int startOffset = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref current) / sizeof(char)); + + if (StringSearchValuesHelper.StartsWith(ref current, span.Length - startOffset, bucket)) + { + return startOffset; + } + } + + if (!Unsafe.IsAddressLessThan(ref current, ref end)) + { + break; + } + + char previous = TCaseSensitivity.TransformInput(current); + char next = TCaseSensitivity.TransformInput(Unsafe.Add(ref current, (uint)hashLength)); + + // Update the hash by removing the previous character and adding the next one. + hash = ((hash - (previous * _hashUpdateMultiplier)) << HashShiftPerElement) + next; + current = ref Unsafe.Add(ref current, 1); + } + } + + return -1; + } + + private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnlySpan span) + { + Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled long inputs."); + + Span upperCase = stackalloc char[MaxInputLength].Slice(0, span.Length); + + int charsWritten = Ordinal.ToUpperOrdinal(span, upperCase); + Debug.Assert(charsWritten == upperCase.Length); + + // CaseSensitive instead of CaseInsensitiveUnicode as we've already done the case conversion. + return IndexOfAnyCore(upperCase); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs new file mode 100644 index 0000000000000..cc985862c95e2 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -0,0 +1,294 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Text; + +namespace System.Buffers +{ + internal static class StringSearchValuesHelper + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool StartsWith(ref char matchStart, int lengthRemaining, string[] candidates) + where TCaseSensitivity : struct, ICaseSensitivity + { + foreach (string candidate in candidates) + { + if (StartsWith(ref matchStart, lengthRemaining, candidate)) + { + return true; + } + } + + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool StartsWith(ref char matchStart, int lengthRemaining, string candidate) + where TCaseSensitivity : struct, ICaseSensitivity + { + if (lengthRemaining < candidate.Length) + { + return false; + } + + return TCaseSensitivity.Equals(ref matchStart, candidate); + } + + public interface IValueLength + { + static abstract bool AtLeast4Chars { get; } + static abstract bool AtLeast8Chars { get; } + } + + public readonly struct ValueLengthLessThan4 : IValueLength + { + public static bool AtLeast4Chars => false; + public static bool AtLeast8Chars => false; + } + + public readonly struct ValueLength4To7 : IValueLength + { + public static bool AtLeast4Chars => true; + public static bool AtLeast8Chars => false; + } + + public readonly struct ValueLength8OrLonger : IValueLength + { + public static bool AtLeast4Chars => true; + public static bool AtLeast8Chars => true; + } + + public interface ICaseSensitivity + { + static abstract char TransformInput(char input); + static abstract Vector128 TransformInput(Vector128 input); + static abstract Vector256 TransformInput(Vector256 input); + static abstract Vector512 TransformInput(Vector512 input); + static abstract bool Equals(ref char matchStart, string candidate); + static abstract bool Equals(ref char matchStart, string candidate) where TValueLength : struct, IValueLength; + } + + public readonly struct CaseSensitive : ICaseSensitivity + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static char TransformInput(char input) => input; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 TransformInput(Vector128 input) => input; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 TransformInput(Vector256 input) => input; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 TransformInput(Vector512 input) => input; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + { + ref char end = ref Unsafe.Add(ref matchStart, candidate.Length); + ref char candidateRef = ref Unsafe.AsRef(candidate.GetPinnableReference()); + + do + { + if (candidateRef != matchStart) + { + return false; + } + + matchStart = ref Unsafe.Add(ref matchStart, 1); + candidateRef = ref Unsafe.Add(ref candidateRef, 1); + } + while (Unsafe.IsAddressLessThan(ref matchStart, ref end)); + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + where TValueLength : struct, IValueLength + { + Debug.Assert(candidate.Length > 1); + Debug.Assert(matchStart == candidate[0], "This should only be called after the first character has been checked"); + + ref byte first = ref Unsafe.As(ref matchStart); + ref byte second = ref Unsafe.As(ref candidate.GetRawStringData()); + nuint byteLength = (nuint)(uint)candidate.Length * 2; + + if (TValueLength.AtLeast8Chars) + { + return SpanHelpers.SequenceEqual(ref first, ref second, byteLength); + } + else if (TValueLength.AtLeast4Chars) + { + nuint offset = byteLength - sizeof(ulong); + ulong differentBits = Unsafe.ReadUnaligned(ref first) - Unsafe.ReadUnaligned(ref second); + differentBits |= Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); + return differentBits == 0; + } + else + { + nuint offset = byteLength - sizeof(uint); + + return Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) + == Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); + } + } + } + + public readonly struct CaseInensitiveAsciiLetters : ICaseSensitivity + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static char TransformInput(char input) => (char)(input & ~0x20); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 TransformInput(Vector128 input) => input & Vector128.Create(unchecked((byte)~0x20)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 TransformInput(Vector256 input) => input & Vector256.Create(unchecked((byte)~0x20)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 TransformInput(Vector512 input) => input & Vector512.Create(unchecked((byte)~0x20)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + { + for (int i = 0; i < candidate.Length; i++) + { + if ((Unsafe.Add(ref matchStart, i) & ~0x20) != candidate[i]) + { + return false; + } + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + where TValueLength : struct, IValueLength + { + Debug.Assert(candidate.Length > 1); + Debug.Assert(candidate.ToUpperInvariant() == candidate); + + if (TValueLength.AtLeast8Chars) + { + return Ascii.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), (uint)candidate.Length); + } + + ref byte first = ref Unsafe.As(ref matchStart); + ref byte second = ref Unsafe.As(ref candidate.GetRawStringData()); + nuint byteLength = (nuint)(uint)candidate.Length * 2; + + if (TValueLength.AtLeast4Chars) + { + nuint offset = byteLength - sizeof(ulong); + ulong differentBits = (Unsafe.ReadUnaligned(ref first) & ~0x20002000200020u) - Unsafe.ReadUnaligned(ref second); + differentBits |= (Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) & ~0x20002000200020u) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); + return differentBits == 0; + } + else + { + nuint offset = byteLength - sizeof(uint); + uint differentBits = (Unsafe.ReadUnaligned(ref first) & ~0x200020u) - Unsafe.ReadUnaligned(ref second); + differentBits |= (Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) & ~0x200020u) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); + return differentBits == 0; + } + } + } + + public readonly struct CaseInensitiveAscii : ICaseSensitivity + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static char TransformInput(char input) => TextInfo.ToUpperAsciiInvariant(input); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 TransformInput(Vector128 input) + { + Vector128 subtraction = Vector128.Create((byte)(128 + 'a')); + Vector128 comparison = Vector128.Create((byte)(128 + 26)); + Vector128 caseConversion = Vector128.Create((byte)0x20); + + Vector128 matches = Vector128.LessThan((input - subtraction).AsSByte(), comparison.AsSByte()).AsByte(); + return input ^ (matches & caseConversion); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 TransformInput(Vector256 input) + { + Vector256 subtraction = Vector256.Create((byte)(128 + 'a')); + Vector256 comparison = Vector256.Create((byte)(128 + 26)); + Vector256 caseConversion = Vector256.Create((byte)0x20); + + Vector256 matches = Vector256.LessThan((input - subtraction).AsSByte(), comparison.AsSByte()).AsByte(); + return input ^ (matches & caseConversion); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 TransformInput(Vector512 input) + { + Vector512 subtraction = Vector512.Create((byte)(128 + 'a')); + Vector512 comparison = Vector512.Create((byte)(128 + 26)); + Vector512 caseConversion = Vector512.Create((byte)0x20); + + Vector512 matches = Vector512.LessThan((input - subtraction).AsSByte(), comparison.AsSByte()).AsByte(); + return input ^ (matches & caseConversion); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + { + for (int i = 0; i < candidate.Length; i++) + { + if (TextInfo.ToUpperAsciiInvariant(Unsafe.Add(ref matchStart, i)) != candidate[i]) + { + return false; + } + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + where TValueLength : struct, IValueLength + { + if (Vector128.IsHardwareAccelerated && TValueLength.AtLeast8Chars) + { + return Ascii.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), (uint)candidate.Length); + } + + return Equals(ref matchStart, candidate); + } + } + + public readonly struct CaseInsensitiveUnicode : ICaseSensitivity + { + public static char TransformInput(char input) => throw new UnreachableException(); + public static Vector128 TransformInput(Vector128 input) => throw new UnreachableException(); + public static Vector256 TransformInput(Vector256 input) => throw new UnreachableException(); + public static Vector512 TransformInput(Vector512 input) => throw new UnreachableException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + { + return Ordinal.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool Equals(ref char matchStart, string candidate) + where TValueLength : struct, IValueLength + { + if (Vector128.IsHardwareAccelerated && TValueLength.AtLeast8Chars) + { + return Ordinal.EqualsIgnoreCase_Vector128(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); + } + + return Ordinal.EqualsIgnoreCase_Scalar(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs new file mode 100644 index 0000000000000..5584d97edc1b9 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace System.Buffers +{ + internal static class TeddyBucketizer + { + public static (Vector512 Low, Vector512 High) GenerateNonBucketizedFingerprint(ReadOnlySpan values, int offset) + { + Debug.Assert(values.Length <= 8); + + Vector128 low = default; + Vector128 high = default; + + for (int i = 0; i < values.Length; i++) + { + string value = values[i]; + + int bit = 1 << i; + + char c = value[offset]; + Debug.Assert(char.IsAscii(c)); + + int lowNibble = c & 0xF; + int highNibble = c >> 4; + + low.SetElementUnsafe(lowNibble, (byte)(low.GetElementUnsafe(lowNibble) | bit)); + high.SetElementUnsafe(highNibble, (byte)(high.GetElementUnsafe(highNibble) | bit)); + } + + return (DuplicateTo512(low), DuplicateTo512(high)); + } + + public static (Vector512 Low, Vector512 High) GenerateBucketizedFingerprint(string[][] valueBuckets, int offset) + { + Debug.Assert(valueBuckets.Length <= 8); + + Vector128 low = default; + Vector128 high = default; + + for (int i = 0; i < valueBuckets.Length; i++) + { + int bit = 1 << i; + + foreach (string value in valueBuckets[i]) + { + char c = value[offset]; + Debug.Assert(char.IsAscii(c)); + + int lowNibble = c & 0xF; + int highNibble = c >> 4; + + low.SetElementUnsafe(lowNibble, (byte)(low.GetElementUnsafe(lowNibble) | bit)); + high.SetElementUnsafe(highNibble, (byte)(high.GetElementUnsafe(highNibble) | bit)); + } + } + + return (DuplicateTo512(low), DuplicateTo512(high)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 DuplicateTo512(Vector128 vector) + { + Vector256 vector256 = Vector256.Create(vector, vector); + return Vector512.Create(vector256, vector256); + } + + public static string[][] Bucketize(ReadOnlySpan values, int bucketCount, int n) + { + Debug.Assert(bucketCount == 8, "This may change if we end up supporting the 'fat Teddy' variant."); + Debug.Assert(values.Length > bucketCount, "Should be using a non-bucketized implementation."); + Debug.Assert(values.Length <= RabinKarp.MaxValues); + + // Stores the offset of the bucket each value should be assigned to. + // This lets us avoid allocating temporary lists to build up each bucket. + Span bucketIndexes = stackalloc int[RabinKarp.MaxValues].Slice(0, values.Length); + + // Group patterns with the same prefix into the same bucket to avoid wasting time during verification steps. + Dictionary prefixToBucket = new(bucketCount); + + int bucketCounter = 0; + + for (int i = 0; i < values.Length; i++) + { + string value = values[i]; + + int prefix = 0; + for (int j = 0; j < n; j++) + { + Debug.Assert(char.IsAscii(value[j])); + prefix = (prefix << 8) | value[j]; + } + + if (!prefixToBucket.TryGetValue(prefix, out int bucketIndex)) + { + // TODO: We currently merge values with different prefixes into buckets randomly (round-robin). + // We could employ a more sophisticated strategy here, e.g. by trying to minimize the number of + // values in each bucket, or by minimizing the PopCount of final merged fingerprints. + // Example of the latter: https://gist.github.com/MihaZupan/831324d1d646b69ae0ba4b54e3446a49 + + bucketIndex = bucketCounter++ % bucketCount; + prefixToBucket.Add(prefix, bucketIndex); + } + + bucketIndexes[i] = bucketIndex; + } + + string[][] buckets = new string[bucketCount][]; + + for (int bucketIndex = 0; bucketIndex < buckets.Length; bucketIndex++) + { + string[] strings = buckets[bucketIndex] = new string[bucketIndexes.Count(bucketIndex)]; + + int count = 0; + for (int i = 0; i < bucketIndexes.Length; i++) + { + if (bucketIndexes[i] == bucketIndex) + { + strings[count++] = values[i]; + } + } + Debug.Assert(count == strings.Length); + } + + return buckets; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs new file mode 100644 index 0000000000000..c2e190026c0c4 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs @@ -0,0 +1,368 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + /// + /// Contains the implementation of core vectorized Teddy matching operations. + /// + /// + /// TODO: Reworded explanation of how the algorithm works. + /// https://github.com/jneem/teddy#teddy-1 is a good starting point. + /// + internal static class TeddyHelper + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public static (Vector128 Result, Vector128 Prev0) ProcessInputN2( + Vector128 input, + Vector128 prev0, + Vector128 n0Low, Vector128 n0High, + Vector128 n1Low, Vector128 n1High) + { + (Vector128 low, Vector128 high) = GetNibbles(input); + + Vector128 match0 = Shuffle(n0Low, n0High, low, high); + Vector128 result1 = Shuffle(n1Low, n1High, low, high); + + Vector128 result0 = RightShift1(prev0, match0); + + Vector128 result = result0 & result1; + + return (result, match0); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public static (Vector256 Result, Vector256 Prev0) ProcessInputN2( + Vector256 input, + Vector256 prev0, + Vector256 n0Low, Vector256 n0High, + Vector256 n1Low, Vector256 n1High) + { + (Vector256 low, Vector256 high) = GetNibbles(input); + + Vector256 match0 = Shuffle(n0Low, n0High, low, high); + Vector256 result1 = Shuffle(n1Low, n1High, low, high); + + Vector256 result0 = RightShift1(prev0, match0); + + Vector256 result = result0 & result1; + + return (result, match0); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + public static (Vector512 Result, Vector512 Prev0) ProcessInputN2( + Vector512 input, + Vector512 prev0, + Vector512 n0Low, Vector512 n0High, + Vector512 n1Low, Vector512 n1High) + { + (Vector512 low, Vector512 high) = GetNibbles(input); + + Vector512 match0 = Shuffle(n0Low, n0High, low, high); + Vector512 result1 = Shuffle(n1Low, n1High, low, high); + + Vector512 result0 = RightShift1(prev0, match0); + + Vector512 result = result0 & result1; + + return (result, match0); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public static (Vector128 Result, Vector128 Prev0, Vector128 Prev1) ProcessInputN3( + Vector128 input, + Vector128 prev0, Vector128 prev1, + Vector128 n0Low, Vector128 n0High, + Vector128 n1Low, Vector128 n1High, + Vector128 n2Low, Vector128 n2High) + { + (Vector128 low, Vector128 high) = GetNibbles(input); + + Vector128 match0 = Shuffle(n0Low, n0High, low, high); + Vector128 match1 = Shuffle(n1Low, n1High, low, high); + Vector128 result2 = Shuffle(n2Low, n2High, low, high); + + Vector128 result0 = RightShift2(prev0, match0); + Vector128 result1 = RightShift1(prev1, match1); + + Vector128 result = result0 & result1 & result2; + + return (result, match0, match1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public static (Vector256 Result, Vector256 Prev0, Vector256 Prev1) ProcessInputN3( + Vector256 input, + Vector256 prev0, Vector256 prev1, + Vector256 n0Low, Vector256 n0High, + Vector256 n1Low, Vector256 n1High, + Vector256 n2Low, Vector256 n2High) + { + (Vector256 low, Vector256 high) = GetNibbles(input); + + Vector256 match0 = Shuffle(n0Low, n0High, low, high); + Vector256 match1 = Shuffle(n1Low, n1High, low, high); + Vector256 result2 = Shuffle(n2Low, n2High, low, high); + + Vector256 result0 = RightShift2(prev0, match0); + Vector256 result1 = RightShift1(prev1, match1); + + Vector256 result = result0 & result1 & result2; + + return (result, match0, match1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + public static (Vector512 Result, Vector512 Prev0, Vector512 Prev1) ProcessInputN3( + Vector512 input, + Vector512 prev0, Vector512 prev1, + Vector512 n0Low, Vector512 n0High, + Vector512 n1Low, Vector512 n1High, + Vector512 n2Low, Vector512 n2High) + { + (Vector512 low, Vector512 high) = GetNibbles(input); + + Vector512 match0 = Shuffle(n0Low, n0High, low, high); + Vector512 match1 = Shuffle(n1Low, n1High, low, high); + Vector512 result2 = Shuffle(n2Low, n2High, low, high); + + Vector512 result0 = RightShift2(prev0, match0); + Vector512 result1 = RightShift1(prev1, match1); + + Vector512 result = result0 & result1 & result2; + + return (result, match0, match1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + [CompExactlyDependsOn(typeof(AdvSimd))] + public static Vector128 LoadAndPack16AsciiChars(ref char source) + { + Vector128 source0 = Vector128.LoadUnsafe(ref source); + Vector128 source1 = Vector128.LoadUnsafe(ref source, (nuint)Vector128.Count); + + return Sse2.IsSupported + ? Sse2.PackUnsignedSaturate(source0.AsInt16(), source1.AsInt16()) + : AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(source0), source1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public static Vector256 LoadAndPack32AsciiChars(ref char source) + { + Vector256 source0 = Vector256.LoadUnsafe(ref source); + Vector256 source1 = Vector256.LoadUnsafe(ref source, (nuint)Vector256.Count); + + Vector256 packed = Avx2.PackUnsignedSaturate(source0.AsInt16(), source1.AsInt16()); + + return PackedSpanHelpers.FixUpPackedVector256Result(packed); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + public static Vector512 LoadAndPack64AsciiChars(ref char source) + { + Vector512 source0 = Vector512.LoadUnsafe(ref source); + Vector512 source1 = Vector512.LoadUnsafe(ref source, (nuint)Vector512.Count); + + Vector512 packed = Avx512BW.PackUnsignedSaturate(source0.AsInt16(), source1.AsInt16()); + + return PackedSpanHelpers.FixUpPackedVector512Result(packed); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + private static (Vector128 Low, Vector128 High) GetNibbles(Vector128 input) + { + // 'low' is not strictly correct here, but we take advantage of Ssse3.Shuffle's behavior + // of doing an implicit 'AND 0xF' in order to skip the redundant AND. + Vector128 low = Ssse3.IsSupported + ? input + : input & Vector128.Create((byte)0xF); + + // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 + Vector128 high = AdvSimd.IsSupported + ? AdvSimd.ShiftRightLogical(input, 4) + : (input.AsInt32() >>> 4).AsByte() & Vector128.Create((byte)0xF); + + return (low, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (Vector256 Low, Vector256 High) GetNibbles(Vector256 input) + { + // 'low' is not strictly correct here, but we take advantage of Avx2.Shuffle's behavior + // of doing an implicit 'AND 0xF' in order to skip the redundant AND. + Vector256 low = input; + + // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 + Vector256 high = (input.AsInt32() >>> 4).AsByte() & Vector256.Create((byte)0xF); + + return (low, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (Vector512 Low, Vector512 High) GetNibbles(Vector512 input) + { + // 'low' is not strictly correct here, but we take advantage of Avx512BW.Shuffle's behavior + // of doing an implicit 'AND 0xF' in order to skip the redundant AND. + Vector512 low = input; + + // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 + Vector512 high = (input.AsInt32() >>> 4).AsByte() & Vector512.Create((byte)0xF); + + return (low, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static Vector128 Shuffle(Vector128 maskLow, Vector128 maskHigh, Vector128 low, Vector128 high) + { + return Vector128.ShuffleUnsafe(maskLow, low) & Vector128.ShuffleUnsafe(maskHigh, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + private static Vector256 Shuffle(Vector256 maskLow, Vector256 maskHigh, Vector256 low, Vector256 high) + { + return Avx2.Shuffle(maskLow, low) & Avx2.Shuffle(maskHigh, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + private static Vector512 Shuffle(Vector512 maskLow, Vector512 maskHigh, Vector512 low, Vector512 high) + { + return Avx512BW.Shuffle(maskLow, low) & Avx512BW.Shuffle(maskHigh, high); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static Vector128 RightShift1(Vector128 left, Vector128 right) + { + // Given input vectors like + // left: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + // right: [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + // We want to shift the last element of left (15) to be the first element of the result + // result: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + + if (Ssse3.IsSupported) + { + return Ssse3.AlignRight(right, left, 15); + } + else + { + Vector128 leftShifted = Vector128.Shuffle(left, Vector128.Create(15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).AsByte()); + return AdvSimd.Arm64.VectorTableLookupExtension(leftShifted, right, Vector128.Create(0xFF, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static Vector128 RightShift2(Vector128 left, Vector128 right) + { + // Given input vectors like + // left: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + // right: [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + // We want to shift the last two elements of left (14, 15) to be the first elements of the result + // result: [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29] + + if (Ssse3.IsSupported) + { + return Ssse3.AlignRight(right, left, 14); + } + else + { + Vector128 leftShifted = Vector128.Shuffle(left, Vector128.Create(14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).AsByte()); + return AdvSimd.Arm64.VectorTableLookupExtension(leftShifted, right, Vector128.Create(0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + private static Vector256 RightShift1(Vector256 left, Vector256 right) + { + // Given input vectors like + // left: 0, 1, 2, 3, 4, 5, ... , 26, 27, 28, 29, 30, [31] + // right: 32, 33, 34, 35, 36, 37, ... , 58, 59, 60, 61, 62, 63 + // We want to shift the last element of left (31) to be the first element of the result + // result: [31], 32, 33, 34, 35, 36, ... , 57, 58, 59, 60, 61, 62 + // + // Avx2.AlignRight acts like two separate Ssse3.AlignRight calls on the lower and upper halves of the source operands. + // Result of Avx2.AlignRight(right, left, 15) is + // lower: [15], 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, + // upper: [31], 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62 + // note how elements at indexes 0 and 16 are off by 16 places. + // We want to read 31 instead of 15 and 47 instead of 31. + // + // To achieve that we create a temporary value where we combine the second half of the first operand and the first half of the second operand (Permute2x128). + // left: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, [ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ] control: (1 << 0) + // right: [ 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 ], 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 control: (2 << 4) + // result: 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, [31], 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, [47] + // This effectively shifts the 0th and 16th element by 16 places (note values 31 and 47). + + Vector256 leftShifted = Avx2.Permute2x128(left, right, (1 << 0) + (2 << 4)); + return Avx2.AlignRight(right, leftShifted, 15); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + private static Vector256 RightShift2(Vector256 left, Vector256 right) + { + // See comments in 'RightShift1(Vector256 left, Vector256 right)' above. + Vector256 leftShifted = Avx2.Permute2x128(left, right, (1 << 0) + (2 << 4)); + return Avx2.AlignRight(right, leftShifted, 14); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + private static Vector512 RightShift1(Vector512 left, Vector512 right) + { + // Given input vectors like + // left: 0, 1, 2, 3, 4, 5, ... , 58, 59, 60, 61, 62, [63] + // right: 64, 65, 66, 67, 68, 69, ... , 122, 123, 124, 125, 126, 127 + // We want to shift the last element of left (63) to be the first element of the result + // result: [63], 64, 65, 66, 67, 68, ... , 121, 122, 123, 124, 125, 126 + // + // Avx512BW.AlignRight acts like four separate Ssse3.AlignRight calls on each 128-bit pair of the of the source operands. + // Result of Avx512BW.AlignRight(right, left, 15) is + // lower: [15], 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, [31], 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, + // upper: [47], 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, [63], 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126 + // note how elements at indexes 0, 16, 32 and 48 are off by 48 places. + // We want to read 63 instead of 15, 79 instead of 31, 95 instead of 47, and 111 instead of 63. + // + // Similar to Avx2 above, we create a temporary value where we shift these positions by 48 places - shift 8-byte values by 6 places (PermuteVar8x64x2). + // The indices vector below could be [6, 7, 8, 9, 10, 11, 12, 13], but we only care about the last byte in each 128-bit block (positions with value 0 don't affect the result). + + Vector512 leftShifted = Avx512F.PermuteVar8x64x2(left.AsInt64(), Vector512.Create(0, 7, 0, 9, 0, 11, 0, 13), right.AsInt64()).AsByte(); + return Avx512BW.AlignRight(right, leftShifted, 15); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + private static Vector512 RightShift2(Vector512 left, Vector512 right) + { + // See comments in 'RightShift1(Vector512 left, Vector512 right)' above. + Vector512 leftShifted = Avx512F.PermuteVar8x64x2(left.AsInt64(), Vector512.Create(0, 7, 0, 9, 0, 11, 0, 13), right.AsInt64()).AsByte(); + return Avx512BW.AlignRight(right, leftShifted, 14); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesFallback.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesFallback.cs new file mode 100644 index 0000000000000..e5cf3b7957cee --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesFallback.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + internal sealed class SingleStringSearchValuesFallback : StringSearchValuesBase + where TIgnoreCase : struct, SearchValues.IRuntimeConst + { + private readonly string _value; + + public SingleStringSearchValuesFallback(string value, HashSet uniqueValues) : base(uniqueValues) + { + _value = value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + TIgnoreCase.Value + ? Ordinal.IndexOfOrdinalIgnoreCase(span, _value) + : span.IndexOf(_value); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs new file mode 100644 index 0000000000000..72b900ac7d31e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -0,0 +1,328 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using static System.Buffers.StringSearchValuesHelper; + +namespace System.Buffers +{ + // Based on SpanHelpers.IndexOf(ref char, int, ref char, int) + // This implementation uses 3 precomputed anchor points when searching. + internal sealed class SingleStringSearchValuesThreeChars : StringSearchValuesBase + where TValueLength : struct, IValueLength + where TCaseSensitivity : struct, ICaseSensitivity + { + private const ushort CaseConversionMask = unchecked((ushort)~0x20); + + private readonly string _value; + private readonly nint _minusValueTailLength; + private readonly nuint _ch2ByteOffset; + private readonly nuint _ch3ByteOffset; + private readonly ushort _ch1; + private readonly ushort _ch2; + private readonly ushort _ch3; + + public SingleStringSearchValuesThreeChars(string value, HashSet uniqueValues) : base(uniqueValues) + { + Debug.Assert(value.Length > 1); + + bool ignoreCase = typeof(TCaseSensitivity) != typeof(CaseSensitive); + + CharacterFrequencyHelper.GetSingleStringMultiCharacterOffsets(value, ignoreCase, out int ch2Offset, out int ch3Offset); + + Debug.Assert(ch3Offset == 0 || ch3Offset > ch2Offset); + + _value = value; + _minusValueTailLength = -(value.Length - 1); + + _ch1 = value[0]; + _ch2 = value[ch2Offset]; + _ch3 = value[ch3Offset]; + + if (ignoreCase) + { + _ch1 &= CaseConversionMask; + _ch2 &= CaseConversionMask; + _ch3 &= CaseConversionMask; + } + + _ch2ByteOffset = (nuint)ch2Offset * 2; + _ch3ByteOffset = (nuint)ch3Offset * 2; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + IndexOf(ref MemoryMarshal.GetReference(span), span.Length); + + private int IndexOf(ref char searchSpace, int searchSpaceLength) + { + ref char searchSpaceStart = ref searchSpace; + + nint searchSpaceMinusValueTailLength = searchSpaceLength + _minusValueTailLength; + + if (!Vector128.IsHardwareAccelerated || searchSpaceMinusValueTailLength < Vector128.Count) + { + goto ShortInput; + } + + nuint ch2ByteOffset = _ch2ByteOffset; + nuint ch3ByteOffset = _ch3ByteOffset; + + if (Vector512.IsHardwareAccelerated && searchSpaceMinusValueTailLength - Vector512.Count >= 0) + { + Vector512 ch1 = Vector512.Create(_ch1); + Vector512 ch2 = Vector512.Create(_ch2); + Vector512 ch3 = Vector512.Create(_ch3); + + ref char lastSearchSpace = ref Unsafe.Add(ref searchSpace, searchSpaceMinusValueTailLength - Vector512.Count); + + while (true) + { + Vector512 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); + + if (result != Vector512.Zero) + { + goto CandidateFound; + } + + LoopFooter: + searchSpace = ref Unsafe.Add(ref searchSpace, Vector512.Count); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpace)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpace, Vector512.Count))) + { + return -1; + } + + searchSpace = ref lastSearchSpace; + } + + continue; + + CandidateFound: + if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + { + return offset; + } + goto LoopFooter; + } + } + else if (Vector256.IsHardwareAccelerated && searchSpaceMinusValueTailLength - Vector256.Count >= 0) + { + Vector256 ch1 = Vector256.Create(_ch1); + Vector256 ch2 = Vector256.Create(_ch2); + Vector256 ch3 = Vector256.Create(_ch3); + + ref char lastSearchSpace = ref Unsafe.Add(ref searchSpace, searchSpaceMinusValueTailLength - Vector256.Count); + + while (true) + { + Vector256 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); + + if (result != Vector256.Zero) + { + goto CandidateFound; + } + + LoopFooter: + searchSpace = ref Unsafe.Add(ref searchSpace, Vector256.Count); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpace)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpace, Vector256.Count))) + { + return -1; + } + + searchSpace = ref lastSearchSpace; + } + + continue; + + CandidateFound: + if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + { + return offset; + } + goto LoopFooter; + } + } + else + { + Vector128 ch1 = Vector128.Create(_ch1); + Vector128 ch2 = Vector128.Create(_ch2); + Vector128 ch3 = Vector128.Create(_ch3); + + ref char lastSearchSpace = ref Unsafe.Add(ref searchSpace, searchSpaceMinusValueTailLength - Vector128.Count); + + while (true) + { + Vector128 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); + + if (result != Vector128.Zero) + { + goto CandidateFound; + } + + LoopFooter: + searchSpace = ref Unsafe.Add(ref searchSpace, Vector128.Count); + + if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpace)) + { + if (Unsafe.AreSame(ref searchSpace, ref Unsafe.Add(ref lastSearchSpace, Vector128.Count))) + { + return -1; + } + + searchSpace = ref lastSearchSpace; + } + + continue; + + CandidateFound: + if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + { + return offset; + } + goto LoopFooter; + } + } + + ShortInput: + string value = _value; + char valueHead = value.GetRawStringData(); + + for (nint i = 0; i < searchSpaceMinusValueTailLength; i++) + { + ref char cur = ref Unsafe.Add(ref searchSpace, i); + + if ((typeof(TCaseSensitivity) == typeof(CaseInsensitiveUnicode) || TCaseSensitivity.TransformInput(cur) == valueHead) && + TCaseSensitivity.Equals(ref cur, value)) + { + return (int)i; + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector128 ch1, Vector128 ch2, Vector128 ch3) + { + if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) + { + Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace)); + Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16()); + Vector128 cmpCh3 = Vector128.Equals(ch3, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16()); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + else + { + Vector128 caseConversion = Vector128.Create(CaseConversionMask); + + Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace) & caseConversion); + Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16() & caseConversion); + Vector128 cmpCh3 = Vector128.Equals(ch3, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16() & caseConversion); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector256 ch1, Vector256 ch2, Vector256 ch3) + { + if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) + { + Vector256 cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref searchSpace)); + Vector256 cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16()); + Vector256 cmpCh3 = Vector256.Equals(ch3, Vector256.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16()); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + else + { + Vector256 caseConversion = Vector256.Create(CaseConversionMask); + + Vector256 cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref searchSpace) & caseConversion); + Vector256 cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16() & caseConversion); + Vector256 cmpCh3 = Vector256.Equals(ch3, Vector256.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16() & caseConversion); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector512 ch1, Vector512 ch2, Vector512 ch3) + { + if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) + { + Vector512 cmpCh1 = Vector512.Equals(ch1, Vector512.LoadUnsafe(ref searchSpace)); + Vector512 cmpCh2 = Vector512.Equals(ch2, Vector512.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16()); + Vector512 cmpCh3 = Vector512.Equals(ch3, Vector512.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16()); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + else + { + Vector512 caseConversion = Vector512.Create(CaseConversionMask); + + Vector512 cmpCh1 = Vector512.Equals(ch1, Vector512.LoadUnsafe(ref searchSpace) & caseConversion); + Vector512 cmpCh2 = Vector512.Equals(ch2, Vector512.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16() & caseConversion); + Vector512 cmpCh3 = Vector512.Equals(ch3, Vector512.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16() & caseConversion); + return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask, out int offsetFromStart) + { + do + { + int bitPos = BitOperations.TrailingZeroCount(mask); + Debug.Assert(bitPos % 2 == 0); + + ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); + + if ((typeof(TCaseSensitivity) == typeof(CaseSensitive) && !TValueLength.AtLeast4Chars) || + TCaseSensitivity.Equals(ref matchRef, _value)) + { + offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); + return true; + } + + mask = BitOperations.ResetLowestSetBit(BitOperations.ResetLowestSetBit(mask)); + } + while (mask != 0); + + offsetFromStart = 0; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mask, out int offsetFromStart) + { + do + { + int bitPos = BitOperations.TrailingZeroCount(mask); + Debug.Assert(bitPos % 2 == 0); + + ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); + + if ((typeof(TCaseSensitivity) == typeof(CaseSensitive) && !TValueLength.AtLeast4Chars) || + TCaseSensitivity.Equals(ref matchRef, _value)) + { + offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); + return true; + } + + mask = BitOperations.ResetLowestSetBit(BitOperations.ResetLowestSetBit(mask)); + } + while (mask != 0); + + offsetFromStart = 0; + return false; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs new file mode 100644 index 0000000000000..83c264fa1d0ee --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -0,0 +1,391 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +using System.Text; +using static System.Buffers.StringSearchValuesHelper; + +namespace System.Buffers +{ + internal static class StringSearchValues + { + private static readonly SearchValues s_asciiLetters = + SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + + public static SearchValues Create(ReadOnlySpan values, bool ignoreCase) + { + if (values.Length == 0) + { + return new EmptySearchValues(); + } + + var uniqueValues = new HashSet(values.Length, ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + + foreach (string value in values) + { + ArgumentNullException.ThrowIfNull(value, nameof(values)); + + uniqueValues.Add(value); + } + + if (uniqueValues.Contains(string.Empty)) + { + return new SingleStringSearchValuesFallback(string.Empty, uniqueValues); + } + + Span normalizedValues = new string[uniqueValues.Count]; + int i = 0; + foreach (string value in uniqueValues) + { + normalizedValues[i++] = NormalizeIfNeeded(value, ignoreCase); + } + Debug.Assert(i == normalizedValues.Length); + + if (normalizedValues.Length == 1) + { + // Avoid the overhead of building the AhoCorasick trie for single-value inputs. + AnalyzeValues(normalizedValues, ref ignoreCase, out bool ascii, out bool asciiLettersOnly, out _, out _); + return CreateForSingleValue(normalizedValues[0], uniqueValues, ignoreCase, ascii, asciiLettersOnly); + } + + // Aho-Corasick's ctor expects values to be sorted by length. + normalizedValues.Sort(static (a, b) => a.Length.CompareTo(b.Length)); + + // We may not end up choosing Aho-Corasick as the implementation, but it has a nice property of + // finding all the unreachable values during the construction stage, so we build the trie early. + HashSet? unreachableValues = null; + var ahoCorasickBuilder = new AhoCorasickBuilder(normalizedValues, ignoreCase, ref unreachableValues); + + if (unreachableValues is not null) + { + // Some values are exact prefixes of other values. + // Exclude those values now to reduce the number of buckets and make verification steps cheaper during searching. + normalizedValues = RemoveUnreachableValues(normalizedValues, unreachableValues); + } + + SearchValues searchValues = CreateFromNormalizedValues(normalizedValues, uniqueValues, ignoreCase, ref ahoCorasickBuilder); + ahoCorasickBuilder.Dispose(); + return searchValues; + + static string NormalizeIfNeeded(string value, bool ignoreCase) + { + if (ignoreCase && (value.AsSpan().ContainsAnyInRange('a', 'z') || !Ascii.IsValid(value))) + { + string upperCase = string.FastAllocateString(value.Length); + int charsWritten = Ordinal.ToUpperOrdinal(value, new Span(ref upperCase.GetRawStringData(), upperCase.Length)); + Debug.Assert(charsWritten == upperCase.Length); + value = upperCase; + } + + return value; + } + + static Span RemoveUnreachableValues(Span values, HashSet unreachableValues) + { + int newCount = 0; + foreach (string value in values) + { + if (!unreachableValues.Contains(value)) + { + values[newCount++] = value; + } + } + + Debug.Assert(newCount <= values.Length - unreachableValues.Count); + Debug.Assert(newCount > 0); + + return values.Slice(0, newCount); + } + } + + private static SearchValues CreateFromNormalizedValues( + ReadOnlySpan values, + HashSet uniqueValues, + bool ignoreCase, + ref AhoCorasickBuilder ahoCorasickBuilder) + { + AnalyzeValues(values, ref ignoreCase, out bool allAscii, out bool asciiLettersOnly, out bool nonAsciiAffectedByCaseConversion, out int minLength); + + if (values.Length == 1) + { + // We may reach this if we've removed unreachable values and ended up with only 1 remaining. + return CreateForSingleValue(values[0], uniqueValues, ignoreCase, allAscii, asciiLettersOnly); + } + + // TODO: Should this be supported anywhere else? + // It may be too niche and too much code for WASM, but AOT with just Vector128 may be interesting. + if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && + TryGetTeddyAcceleratedValues(values, uniqueValues, ignoreCase, allAscii, asciiLettersOnly, nonAsciiAffectedByCaseConversion, minLength) is { } searchValues) + { + return searchValues; + } + + AhoCorasick ahoCorasick = ahoCorasickBuilder.Build(); + + if (!ignoreCase) + { + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + } + + if (nonAsciiAffectedByCaseConversion) + { + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + } + + if (asciiLettersOnly) + { + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + } + + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + + static SearchValues PickAhoCorasickImplementation(AhoCorasick ahoCorasick, HashSet uniqueValues) + where TCaseSensitivity : struct, ICaseSensitivity + { + return ahoCorasick.ShouldUseAsciiFastScan + ? new StringSearchValuesAhoCorasick(ahoCorasick, uniqueValues) + : new StringSearchValuesAhoCorasick(ahoCorasick, uniqueValues); + } + } + + private static SearchValues? TryGetTeddyAcceleratedValues( + ReadOnlySpan values, + HashSet uniqueValues, + bool ignoreCase, + bool allAscii, + bool asciiLettersOnly, + bool nonAsciiAffectedByCaseConversion, + int minLength) + { + if (minLength == 1) + { + // An 'N=1' implementation is possible, but callers should + // consider using SearchValues instead in such cases. + // It can be added if Regex ends up running into this case. + return null; + } + + if (values.Length > RabinKarp.MaxValues) + { + // The more values we have, the higher the chance of hash/fingerprint collisions. + // To avoid spending too much time in verification steps, fallback to Aho-Corasick which guarantees O(n). + // If it turns out that this limit is commonly exceeded, we can tweak the number of buckets + // in the implementation, or use different variants depending on input. + return null; + } + + int n = minLength == 2 ? 2 : 3; + + if (Ssse3.IsSupported) + { + foreach (string value in values) + { + if (value.AsSpan(0, n).Contains('\0')) + { + // If we let null chars through here, Teddy would still work correctly, but it + // would hit more false positives that the verification step would have to rule out. + // While we could flow a generic flag like Ssse3AndWasmHandleZeroInNeedle through, + // we expect such values to be rare enough that introducing more code is not worth it. + return null; + } + } + } + + // Even if the values contain non-ASCII chars, we may be able to use Teddy as long as the + // first N characters are ASCII. + if (!allAscii) + { + foreach (string value in values) + { + if (!Ascii.IsValid(value.AsSpan(0, n))) + { + // A vectorized implementation for non-ASCII values is possible. + // It can be added if it turns out to be a common enough scenario. + return null; + } + } + } + + if (!ignoreCase) + { + return PickTeddyImplementation(values, uniqueValues, n); + } + + if (asciiLettersOnly) + { + return PickTeddyImplementation(values, uniqueValues, n); + } + + // Even if the whole value isn't ASCII letters only, we can still use a faster approach + // for the vectorized part as long as the first N characters are. + bool asciiStartLettersOnly = true; + bool asciiStartUnaffectedByCaseConversion = true; + + foreach (string value in values) + { + ReadOnlySpan slice = value.AsSpan(0, n); + asciiStartLettersOnly = asciiStartLettersOnly && !slice.ContainsAnyExcept(s_asciiLetters); + asciiStartUnaffectedByCaseConversion = asciiStartUnaffectedByCaseConversion && !slice.ContainsAny(s_asciiLetters); + } + + Debug.Assert(!(asciiStartLettersOnly && asciiStartUnaffectedByCaseConversion)); + + if (asciiStartUnaffectedByCaseConversion) + { + return nonAsciiAffectedByCaseConversion + ? PickTeddyImplementation(values, uniqueValues, n) + : PickTeddyImplementation(values, uniqueValues, n); + } + + if (nonAsciiAffectedByCaseConversion) + { + return asciiStartLettersOnly + ? PickTeddyImplementation(values, uniqueValues, n) + : PickTeddyImplementation(values, uniqueValues, n); + } + + return asciiStartLettersOnly + ? PickTeddyImplementation(values, uniqueValues, n) + : PickTeddyImplementation(values, uniqueValues, n); + } + + private static SearchValues PickTeddyImplementation( + ReadOnlySpan values, + HashSet uniqueValues, + int n) + where TStartCaseSensitivity : struct, ICaseSensitivity + where TCaseSensitivity : struct, ICaseSensitivity + { + Debug.Assert(typeof(TStartCaseSensitivity) != typeof(CaseInsensitiveUnicode)); + Debug.Assert(values.Length > 1); + Debug.Assert(n is 2 or 3); + + if (values.Length > 8) + { + string[][] buckets = TeddyBucketizer.Bucketize(values, bucketCount: 8, n); + + // TODO: Should we bail if we encounter a bad bucket distributions? + + // TODO: We don't have to pick the first N characters for the fingerprint. + // Different offset selection can noticeably improve throughput (e.g. 2x). + + return n == 2 + ? new AsciiStringSearchValuesTeddyBucketizedN2(buckets, values, uniqueValues) + : new AsciiStringSearchValuesTeddyBucketizedN3(buckets, values, uniqueValues); + } + else + { + return n == 2 + ? new AsciiStringSearchValuesTeddyNonBucketizedN2(values, uniqueValues) + : new AsciiStringSearchValuesTeddyNonBucketizedN3(values, uniqueValues); + } + } + + private static SearchValues CreateForSingleValue( + string value, + HashSet uniqueValues, + bool ignoreCase, + bool allAscii, + bool asciiLettersOnly) + { + // We make use of optimizations that may overflow on 32bit systems for long values. + int maxLength = IntPtr.Size == 4 ? 1_000_000_000 : int.MaxValue; + + if (Vector128.IsHardwareAccelerated && value.Length > 1 && value.Length <= maxLength) + { + SearchValues? searchValues = value.Length switch + { + < 4 => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), + < 8 => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), + _ => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), + }; + + if (searchValues is not null) + { + return searchValues; + } + } + + return ignoreCase + ? new SingleStringSearchValuesFallback(value, uniqueValues) + : new SingleStringSearchValuesFallback(value, uniqueValues); + } + + private static SearchValues? TryCreateSingleValuesThreeChars( + string value, + HashSet uniqueValues, + bool ignoreCase, + bool allAscii, + bool asciiLettersOnly) + where TValueLength : struct, IValueLength + { + if (!ignoreCase) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + if (asciiLettersOnly) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + if (allAscii) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + // When ignoring casing, all anchor chars we search for must be ASCII. + if (char.IsAscii(value[0]) && value.AsSpan().LastIndexOfAnyInRange((char)0, (char)127) > 0) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + return null; + } + + private static void AnalyzeValues( + ReadOnlySpan values, + ref bool ignoreCase, + out bool allAscii, + out bool asciiLettersOnly, + out bool nonAsciiAffectedByCaseConversion, + out int minLength) + { + allAscii = true; + asciiLettersOnly = true; + minLength = int.MaxValue; + + foreach (string value in values) + { + allAscii = allAscii && Ascii.IsValid(value); + asciiLettersOnly = asciiLettersOnly && !value.AsSpan().ContainsAnyExcept(s_asciiLetters); + minLength = Math.Min(minLength, value.Length); + } + + // TODO: Not all characters participate in Unicode case conversion. + // If we can determine that none of the non-ASCII characters do, we can make searching faster + // by using the same paths as we do for ASCII-only values. + nonAsciiAffectedByCaseConversion = ignoreCase && !allAscii; + + // If all the characters in values are unaffected by casing, we can avoid the ignoreCase overhead. + if (ignoreCase && !nonAsciiAffectedByCaseConversion && !asciiLettersOnly) + { + ignoreCase = false; + + foreach (string value in values) + { + if (value.AsSpan().ContainsAny(s_asciiLetters)) + { + ignoreCase = true; + break; + } + } + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesAhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesAhoCorasick.cs new file mode 100644 index 0000000000000..c6656d55c9f97 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesAhoCorasick.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + internal sealed class StringSearchValuesAhoCorasick : StringSearchValuesBase + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TFastScanVariant : struct, AhoCorasick.IFastScan + { + private readonly AhoCorasick _ahoCorasick; + + public StringSearchValuesAhoCorasick(AhoCorasick ahoCorasick, HashSet uniqueValues) : base(uniqueValues) => + _ahoCorasick = ahoCorasick; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyMultiString(ReadOnlySpan span) => + _ahoCorasick.IndexOfAny(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesBase.cs new file mode 100644 index 0000000000000..b069c235e4d23 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesBase.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Buffers +{ + /// + /// Implements the base {Last}IndexOfAny{Except} operations. + /// While these operations are exposed such that you can call string[].IndexOfAny(searchValues), + /// they are not expected to be used in performance-critical paths. + /// is the main + /// reason why someone would create an instance of . + /// + internal abstract class StringSearchValuesBase : SearchValues + { + private readonly HashSet _uniqueValues; + + public StringSearchValuesBase(HashSet uniqueValues) => + _uniqueValues = uniqueValues; + + internal sealed override bool ContainsCore(string value) => + _uniqueValues.Contains(value); + + internal sealed override string[] GetValues() + { + string[] values = new string[_uniqueValues.Count]; + _uniqueValues.CopyTo(values); + return values; + } + + internal sealed override int IndexOfAny(ReadOnlySpan span) => + IndexOfAny(span); + + internal sealed override int IndexOfAnyExcept(ReadOnlySpan span) => + IndexOfAny(span); + + internal sealed override int LastIndexOfAny(ReadOnlySpan span) => + LastIndexOfAny(span); + + internal sealed override int LastIndexOfAnyExcept(ReadOnlySpan span) => + LastIndexOfAny(span); + + private int IndexOfAny(ReadOnlySpan span) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + for (int i = 0; i < span.Length; i++) + { + if (TNegator.NegateIfNeeded(_uniqueValues.Contains(span[i]))) + { + return i; + } + } + + return -1; + } + + private int LastIndexOfAny(ReadOnlySpan span) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + for (int i = span.Length - 1; i >= 0; i--) + { + if (TNegator.NegateIfNeeded(_uniqueValues.Contains(span[i]))) + { + return i; + } + } + + return -1; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesRabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesRabinKarp.cs new file mode 100644 index 0000000000000..993f2814bfa90 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValuesRabinKarp.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + internal abstract class StringSearchValuesRabinKarp : StringSearchValuesBase + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + private readonly RabinKarp _rabinKarp; + + public StringSearchValuesRabinKarp(ReadOnlySpan values, HashSet uniqueValues) : base(uniqueValues) => + _rabinKarp = new RabinKarp(values); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected int ShortInputFallback(ReadOnlySpan span) => + _rabinKarp.IndexOfAny(span); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index ad91104507a39..fca176fe43812 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -1264,7 +1264,7 @@ private static int ComputeFirstIndexOverlapped(ref short searchSpace, ref short [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] - private static Vector256 FixUpPackedVector256Result(Vector256 result) + internal static Vector256 FixUpPackedVector256Result(Vector256 result) { Debug.Assert(Avx2.IsSupported); // Avx2.PackUnsignedSaturate(Vector256.Create((short)1), Vector256.Create((short)2)) will result in @@ -1276,14 +1276,12 @@ private static Vector256 FixUpPackedVector256Result(Vector256 result [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx512F))] - private static Vector512 FixUpPackedVector512Result(Vector512 result) + internal static Vector512 FixUpPackedVector512Result(Vector512 result) { Debug.Assert(Avx512F.IsSupported); - // Avx512BW.PackUnsignedSaturate(Vector512.Create((short)1), Vector512.Create((short)2)) will result in - // 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 - // We want to swap the X and Y bits - // 1, 1, 1, 1, 1, 1, 1, 1, X, X, X, X, X, X, X, X, Y, Y, Y, Y, Y, Y, Y, Y, 2, 2, 2, 2, 2, 2, 2, 2 - return Avx512F.PermuteVar8x64(result.AsInt64(), Vector512.Create((long)0, 2, 4, 6, 1, 3, 5, 7)).AsByte(); + // Avx512BW.PackUnsignedSaturate will interleave the inputs in 8-byte blocks. + // We want to preserve the order of the two input vectors, so we deinterleave the packed value. + return Avx512F.PermuteVar8x64(result.AsInt64(), Vector512.Create(0, 2, 4, 6, 1, 3, 5, 7)).AsByte(); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs index 31df908d9afb0..1aa19c95adc94 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs @@ -189,6 +189,10 @@ public static bool EqualsIgnoreCase(ReadOnlySpan left, ReadOnlySpan => left.Length == right.Length && EqualsIgnoreCase>(ref Unsafe.As(ref MemoryMarshal.GetReference(left)), ref Unsafe.As(ref MemoryMarshal.GetReference(right)), (uint)right.Length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool EqualsIgnoreCase(ref char left, ref char right, nuint length) => + EqualsIgnoreCase>(ref Unsafe.As(ref left), ref Unsafe.As(ref right), length); + private static bool EqualsIgnoreCase(ref TLeft left, ref TRight right, nuint length) where TLeft : unmanaged, INumberBase where TRight : unmanaged, INumberBase diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 97ef4d74a4450..a992266497707 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -7391,6 +7391,7 @@ public static class SearchValues { public static System.Buffers.SearchValues Create(System.ReadOnlySpan values) { throw null; } public static System.Buffers.SearchValues Create(System.ReadOnlySpan values) { throw null; } + public static System.Buffers.SearchValues Create(System.ReadOnlySpan values, System.StringComparison comparisonType) { throw null; } } public partial interface IPinnable { From c56168d0e77d9385f44a3f99ffcd2fc9e20d6cad Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 01:21:20 +0200 Subject: [PATCH 02/43] Fix AhoCorasickNode DebuggerDisplay --- .../System.Private.CoreLib.Shared.projitems | 1 + .../Strings/Helpers/AhoCorasick.cs | 180 +--------------- .../Strings/Helpers/AhoCorasickBuilder.cs | 12 +- .../Strings/Helpers/AhoCorasickNode.cs | 201 ++++++++++++++++++ 4 files changed, 214 insertions(+), 180 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 32e726a203cc9..a59be71ed501b 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -439,6 +439,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index bcc7c84bc48c8..e01ceaf685b5e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -1,9 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -18,11 +16,11 @@ namespace System.Buffers /// internal readonly struct AhoCorasick { - private readonly Node[] _nodes; + private readonly AhoCorasickNode[] _nodes; private readonly Vector256 _startingCharsAsciiBitmap; private readonly int _maxValueLength; // Only used by the NLS fallback - public AhoCorasick(Node[] nodes, Vector256 startingAsciiBitmap, int maxValueLength) + public AhoCorasick(AhoCorasickNode[] nodes, Vector256 startingAsciiBitmap, int maxValueLength) { _nodes = nodes; _startingCharsAsciiBitmap = startingAsciiBitmap; @@ -88,7 +86,7 @@ private readonly int IndexOfAnyCore(ReadOnly throw new UnreachableException(); } - ref Node nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); + ref AhoCorasickNode nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); int nodeIndex = 0; int result = -1; int i = 0; @@ -131,7 +129,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), while (true) { Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); - ref Node node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); + ref AhoCorasickNode node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); if (node.TryGetChild(c, out int childIndex)) { @@ -180,7 +178,7 @@ private readonly int IndexOfAnyCaseInsensitiveUnicodeIcuOrInvariant(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), while (true) { Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); - ref Node node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); + ref AhoCorasickNode node = ref Unsafe.Add(ref nodes, (uint)nodeIndex); if (node.TryGetChild(c, out int childIndex)) { @@ -378,171 +376,5 @@ public interface IFastScan { } public readonly struct IndexOfAnyAsciiFastScan : IFastScan { } public readonly struct NoFastScan : IFastScan { } - - [DebuggerDisplay("MatchLength={MatchLength} SuffixLink={SuffixLink} ChildrenCount={(_children?.Count ?? 0) + (_firstChildChar < 0 ? 0 : 1)}")] - public struct Node - { - private static object EmptyChildrenSentinel => Array.Empty(); - - public int SuffixLink; - public int MatchLength; - - private int _firstChildChar; - private int _firstChildIndex; - private object _children; // Either Dictionary or int[] - - public Node() - { - _firstChildChar = -1; - _children = EmptyChildrenSentinel; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly bool TryGetChild(char c, out int index) - { - if (_firstChildChar == c) - { - index = _firstChildIndex; - return true; - } - - object children = _children; - Debug.Assert(children is int[] || children is Dictionary); - - if (children.GetType() == typeof(int[])) - { - int[] table = Unsafe.As(children); - if (c < (uint)table.Length) - { - index = table[c]; - if (index >= 0) - { - return true; - } - } - } - else - { - return Unsafe.As>(children).TryGetValue(c, out index); - } - - index = 0; - return false; - } - - public void AddChild(char c, int index) - { - if (_firstChildChar < 0) - { - _firstChildChar = c; - _firstChildIndex = index; - } - else - { - if (ReferenceEquals(_children, EmptyChildrenSentinel)) - { - _children = new Dictionary(); - } - - ((Dictionary)_children).Add(c, index); - } - } - - public readonly void AddChildrenToQueue(Queue<(char Char, int Index)> queue) - { - if (_firstChildChar >= 0) - { - queue.Enqueue(((char)_firstChildChar, _firstChildIndex)); - - if (_children is Dictionary children) - { - foreach ((char childChar, int childIndex) in children) - { - queue.Enqueue((childChar, childIndex)); - } - } - } - } - - public void OptimizeChildren() - { - if (_children is Dictionary children) - { - children.Add((char)_firstChildChar, _firstChildIndex); - - float frequency = -2; - - foreach ((char childChar, int childIndex) in children) - { - float newFrequency = char.IsAscii(childChar) ? CharacterFrequencyHelper.AsciiFrequency[childChar] : -1; - - if (newFrequency > frequency) - { - frequency = newFrequency; - _firstChildChar = childChar; - _firstChildIndex = childIndex; - } - } - - children.Remove((char)_firstChildChar); - - if (TryCreateJumpTable(children, out int[]? table)) - { - _children = table; - } - } - - static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true)] out int[]? table) - { - // Sacrifice some memory usage in exchange for faster lookup performance - const int AcceptableSizeMultiplier = 2; - - Debug.Assert(children.Count > 0); - - int maxValue = -1; - - foreach ((char childChar, _) in children) - { - maxValue = Math.Max(maxValue, childChar); - } - - int tableSize = TableSizeEstimate(maxValue); - int dictionarySize = DictionarySizeEstimate(children.Count); - - if (tableSize > dictionarySize * AcceptableSizeMultiplier) - { - // We would have a lot of empty entries. Avoid wasting too much memory. - table = null; - return false; - } - - table = new int[maxValue + 1]; - Array.Fill(table, -1); - - foreach ((char childChar, int childIndex) in children) - { - table[childChar] = childIndex; - } - - return true; - - static int TableSizeEstimate(int maxValue) - { - return 32 + (maxValue * 4); - } - - static int DictionarySizeEstimate(int childCount) - { - return childCount switch - { - < 4 => 192, - < 8 => 272, - < 12 => 352, - _ => childCount * 25 - }; - } - } - } - } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs index 951ee9bc5417e..d625f12d70e74 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs @@ -15,7 +15,7 @@ internal ref struct AhoCorasickBuilder { private readonly ReadOnlySpan _values; private readonly bool _ignoreCase; - private ValueListBuilder _nodes; + private ValueListBuilder _nodes; private ValueListBuilder _parents; private Vector256 _startingCharsAsciiBitmap; private int _maxValueLength; // Only used by the NLS fallback @@ -65,13 +65,13 @@ public void Dispose() private void BuildTrie(ref HashSet? unreachableValues) { - _nodes.Append(new AhoCorasick.Node()); + _nodes.Append(new AhoCorasickNode()); _parents.Append(0); foreach (string value in _values) { int nodeIndex = 0; - ref AhoCorasick.Node node = ref _nodes[nodeIndex]; + ref AhoCorasickNode node = ref _nodes[nodeIndex]; for (int i = 0; i < value.Length; i++) { @@ -81,7 +81,7 @@ private void BuildTrie(ref HashSet? unreachableValues) { childIndex = _nodes.Length; node.AddChild(c, childIndex); - _nodes.Append(new AhoCorasick.Node()); + _nodes.Append(new AhoCorasickNode()); _parents.Append(nodeIndex); } @@ -115,7 +115,7 @@ private void AddSuffixLinks() while (queue.TryDequeue(out (char Char, int Index) trieNode)) { - ref AhoCorasick.Node node = ref _nodes[trieNode.Index]; + ref AhoCorasickNode node = ref _nodes[trieNode.Index]; int parent = _parents[trieNode.Index]; int suffixLink = _nodes[parent].SuffixLink; @@ -123,7 +123,7 @@ private void AddSuffixLinks() { while (suffixLink >= 0) { - ref AhoCorasick.Node suffixNode = ref _nodes[suffixLink]; + ref AhoCorasickNode suffixNode = ref _nodes[suffixLink]; if (suffixNode.TryGetChild(trieNode.Char, out int childSuffixLink)) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs new file mode 100644 index 0000000000000..55746cab54654 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs @@ -0,0 +1,201 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers +{ + [DebuggerDisplay("{DebuggerDisplay,nq}")] + internal struct AhoCorasickNode + { + private static object EmptyChildrenSentinel => Array.Empty(); + + public int SuffixLink; + public int MatchLength; + + private int _firstChildChar; + private int _firstChildIndex; + private object _children; // Either int[] or Dictionary + + public AhoCorasickNode() + { + _firstChildChar = -1; + _children = EmptyChildrenSentinel; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TryGetChild(char c, out int index) + { + if (_firstChildChar == c) + { + index = _firstChildIndex; + return true; + } + + object children = _children; + Debug.Assert(children is int[] || children is Dictionary); + + if (children.GetType() == typeof(int[])) + { + int[] table = Unsafe.As(children); + if (c < (uint)table.Length) + { + index = table[c]; + if (index >= 0) + { + return true; + } + } + } + else + { + return Unsafe.As>(children).TryGetValue(c, out index); + } + + index = 0; + return false; + } + + public void AddChild(char c, int index) + { + if (_firstChildChar < 0) + { + _firstChildChar = c; + _firstChildIndex = index; + } + else + { + if (ReferenceEquals(_children, EmptyChildrenSentinel)) + { + _children = new Dictionary(); + } + + ((Dictionary)_children).Add(c, index); + } + } + + public readonly void AddChildrenToQueue(Queue<(char Char, int Index)> queue) + { + if (_firstChildChar >= 0) + { + queue.Enqueue(((char)_firstChildChar, _firstChildIndex)); + + if (_children is Dictionary children) + { + foreach ((char childChar, int childIndex) in children) + { + queue.Enqueue((childChar, childIndex)); + } + } + } + } + + public void OptimizeChildren() + { + if (_children is Dictionary children) + { + children.Add((char)_firstChildChar, _firstChildIndex); + + float frequency = -2; + + foreach ((char childChar, int childIndex) in children) + { + float newFrequency = char.IsAscii(childChar) ? CharacterFrequencyHelper.AsciiFrequency[childChar] : -1; + + if (newFrequency > frequency) + { + frequency = newFrequency; + _firstChildChar = childChar; + _firstChildIndex = childIndex; + } + } + + children.Remove((char)_firstChildChar); + + if (TryCreateJumpTable(children, out int[]? table)) + { + _children = table; + } + } + + static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true)] out int[]? table) + { + // Sacrifice some memory usage in exchange for faster lookup performance + const int AcceptableSizeMultiplier = 2; + + Debug.Assert(children.Count > 0); + + int maxValue = -1; + + foreach ((char childChar, _) in children) + { + maxValue = Math.Max(maxValue, childChar); + } + + int tableSize = TableSizeEstimate(maxValue); + int dictionarySize = DictionarySizeEstimate(children.Count); + + if (tableSize > dictionarySize * AcceptableSizeMultiplier) + { + // We would have a lot of empty entries. Avoid wasting too much memory. + table = null; + return false; + } + + table = new int[maxValue + 1]; + Array.Fill(table, -1); + + foreach ((char childChar, int childIndex) in children) + { + table[childChar] = childIndex; + } + + return true; + + static int TableSizeEstimate(int maxValue) + { + return 32 + (maxValue * 4); + } + + static int DictionarySizeEstimate(int childCount) + { + return childCount switch + { + < 4 => 192, + < 8 => 272, + < 12 => 352, + _ => childCount * 25 + }; + } + } + } + + private readonly string DebuggerDisplay + { + get + { + int count = _firstChildChar >= 0 ? 1 : 0; + + if (_children is int[] table) + { + foreach (int index in table) + { + if (index >= 0) + { + count++; + } + } + } + else + { + count += ((Dictionary)_children).Count; + } + + return $"MatchLength={MatchLength} SuffixLink={SuffixLink} ChildrenCount={count}"; + } + } + } +} From 68d3dfdb91ef63d968d68b37b5a7955636aa851e Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 15:08:50 +0200 Subject: [PATCH 03/43] Typos & rewordings --- .../src/Resources/Strings.resx | 2 +- .../src/System/MemoryExtensions.cs | 10 +++++----- .../AsciiStringSearchValuesTeddyBase.cs | 16 ++++++--------- .../Strings/Helpers/AhoCorasick.cs | 8 ++++---- .../Helpers/StringSearchValuesHelper.cs | 4 ++-- .../Strings/StringSearchValues.cs | 20 +++++++++---------- 6 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 91da334936796..11abff4d13cb9 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -4245,6 +4245,6 @@ String length exceeded supported range. - SearchValues<string> supports only Ordinal and OrdinalIgnoreCase StringComparison semantics. + SearchValues<string> supports only StringComparison.Ordinal and StringComparison.OrdinalIgnoreCase. diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 17804a753ef6b..df69789583143 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -457,7 +457,7 @@ public static bool ContainsAnyExceptInRange(this Span span, T lowInclusive ContainsAnyExceptInRange((ReadOnlySpan)span, lowInclusive, highInclusive); /// - /// Searches for any occurance of the specified or , and returns true if found. If not found, returns false. + /// Searches for any occurrence of the specified or , and returns true if found. If not found, returns false. /// /// The span to search. /// One of the values to search for. @@ -467,7 +467,7 @@ public static bool ContainsAny(this ReadOnlySpan span, T value0, T value1) IndexOfAny(span, value0, value1) >= 0; /// - /// Searches for any occurance of the specified , , or , and returns true if found. If not found, returns false. + /// Searches for any occurrence of the specified , , or , and returns true if found. If not found, returns false. /// /// The span to search. /// One of the values to search for. @@ -478,7 +478,7 @@ public static bool ContainsAny(this ReadOnlySpan span, T value0, T value1, IndexOfAny(span, value0, value1, value2) >= 0; /// - /// Searches for any occurance of any of the specified and returns true if found. If not found, returns false. + /// Searches for any occurrence of any of the specified and returns true if found. If not found, returns false. /// /// The span to search. /// The set of values to search for. @@ -487,7 +487,7 @@ public static bool ContainsAny(this ReadOnlySpan span, ReadOnlySpan val IndexOfAny(span, values) >= 0; /// - /// Searches for any occurance of any of the specified and returns true if found. If not found, returns false. + /// Searches for any occurrence of any of the specified and returns true if found. If not found, returns false. /// /// The span to search. /// The set of values to search for. @@ -496,7 +496,7 @@ public static bool ContainsAny(this ReadOnlySpan span, SearchValues val IndexOfAny(span, values) >= 0; /// - /// Searches for any occurance of any of the specified substring and returns true if found. If not found, returns false. + /// Searches for any occurrence of any of the specified substring and returns true if found. If not found, returns false. /// /// The span to search. /// The set of values to search for. diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index 8a51d58106f54..a5b1c8113f256 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -34,10 +34,8 @@ private readonly Vector512 protected AsciiStringSearchValuesTeddyBase(ReadOnlySpan values, HashSet uniqueValues, int n) : base(values, uniqueValues) { - if (TBucketized.Value) - { - throw new UnreachableException(); - } + Debug.Assert(!TBucketized.Value); + Debug.Assert(n is 2 or 3); _buckets = new EightPackedReferences(MemoryMarshal.CreateReadOnlySpan( ref Unsafe.As(ref MemoryMarshal.GetReference(values)), @@ -54,10 +52,8 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), protected AsciiStringSearchValuesTeddyBase(string[][] buckets, ReadOnlySpan values, HashSet uniqueValues, int n) : base(values, uniqueValues) { - if (!TBucketized.Value) - { - throw new UnreachableException(); - } + Debug.Assert(TBucketized.Value); + Debug.Assert(n is 2 or 3); _buckets = new EightPackedReferences(buckets); @@ -86,7 +82,7 @@ protected int IndexOfAnyN2(ReadOnlySpan span) { return IndexOfAnyN2Avx2(span); } -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough +#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough return IndexOfAnyN2Vector128(span); } @@ -107,7 +103,7 @@ protected int IndexOfAnyN3(ReadOnlySpan span) { return IndexOfAnyN3Avx2(span); } -#pragma warning disable IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough +#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough return IndexOfAnyN3Vector128(span); } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index e01ceaf685b5e..7ac9c2afe6960 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -39,10 +39,10 @@ public readonly bool ShouldUseAsciiFastScan // the ASCII fast scan may end up performing worse than checking one character at a time. // Avoid using this optimization if the combined frequency of starting chars is too high. - // Combined frequency of - // - All digits is ~ 5 - // - All lowercase letters is ~ 57.2 - // - All uppercase letters is ~ 7.4 + // Combined frequency of characters based on CharacterFrequencyHelper.AsciiFrequency: + // - All digits is ~ 5 % + // - All lowercase letters is ~ 57.2 % + // - All uppercase letters is ~ 7.4 % const float MaxCombinedFrequency = 50f; float frequency = 0; diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index cc985862c95e2..8c38d7425db40 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -139,7 +139,7 @@ public static bool Equals(ref char matchStart, string candidate) } } - public readonly struct CaseInensitiveAsciiLetters : ICaseSensitivity + public readonly struct CaseInsensitiveAsciiLetters : ICaseSensitivity { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static char TransformInput(char input) => (char)(input & ~0x20); @@ -200,7 +200,7 @@ public static bool Equals(ref char matchStart, string candidate) } } - public readonly struct CaseInensitiveAscii : ICaseSensitivity + public readonly struct CaseInsensitiveAscii : ICaseSensitivity { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static char TransformInput(char input) => TextInfo.ToUpperAsciiInvariant(input); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 83c264fa1d0ee..7c42277f011de 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -139,10 +139,10 @@ private static SearchValues CreateFromNormalizedValues( if (asciiLettersOnly) { - return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); } - return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); static SearchValues PickAhoCorasickImplementation(AhoCorasick ahoCorasick, HashSet uniqueValues) where TCaseSensitivity : struct, ICaseSensitivity @@ -218,7 +218,7 @@ static SearchValues PickAhoCorasickImplementation(AhoC if (asciiLettersOnly) { - return PickTeddyImplementation(values, uniqueValues, n); + return PickTeddyImplementation(values, uniqueValues, n); } // Even if the whole value isn't ASCII letters only, we can still use a faster approach @@ -239,19 +239,19 @@ static SearchValues PickAhoCorasickImplementation(AhoC { return nonAsciiAffectedByCaseConversion ? PickTeddyImplementation(values, uniqueValues, n) - : PickTeddyImplementation(values, uniqueValues, n); + : PickTeddyImplementation(values, uniqueValues, n); } if (nonAsciiAffectedByCaseConversion) { return asciiStartLettersOnly - ? PickTeddyImplementation(values, uniqueValues, n) - : PickTeddyImplementation(values, uniqueValues, n); + ? PickTeddyImplementation(values, uniqueValues, n) + : PickTeddyImplementation(values, uniqueValues, n); } return asciiStartLettersOnly - ? PickTeddyImplementation(values, uniqueValues, n) - : PickTeddyImplementation(values, uniqueValues, n); + ? PickTeddyImplementation(values, uniqueValues, n) + : PickTeddyImplementation(values, uniqueValues, n); } private static SearchValues PickTeddyImplementation( @@ -331,12 +331,12 @@ private static SearchValues CreateForSingleValue( if (asciiLettersOnly) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value, uniqueValues); } if (allAscii) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value, uniqueValues); } // When ignoring casing, all anchor chars we search for must be ASCII. From 515104cfc722bb73de627c0b86a7552808cdede3 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 15:15:35 +0200 Subject: [PATCH 04/43] Use [InlineArray] --- .../Strings/Helpers/EightPackedReferences.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs index 16ca38d813375..43192751a9d70 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs @@ -3,21 +3,13 @@ using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace System.Buffers { - [StructLayout(LayoutKind.Sequential)] + [InlineArray(8)] internal readonly struct EightPackedReferences { private readonly object? _ref0; - private readonly object? _ref1; - private readonly object? _ref2; - private readonly object? _ref3; - private readonly object? _ref4; - private readonly object? _ref5; - private readonly object? _ref6; - private readonly object? _ref7; public EightPackedReferences(ReadOnlySpan values) { From 2a5bef551c293080ffbde13fd5a7c9b0f6ebd3c7 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 21:44:49 +0200 Subject: [PATCH 05/43] Simplify NLS variant of AhoCorasick --- .../src/System/Globalization/TextInfo.cs | 10 ++ .../Strings/Helpers/AhoCorasick.cs | 106 ++++-------------- 2 files changed, 34 insertions(+), 82 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs index eca1831c625bf..172778324166d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs @@ -190,6 +190,16 @@ private unsafe char ChangeCase(char c, bool toUpper) return dst; } + internal static char ToUpperOrdinalNls(char c) + { + if (char.IsAscii(c)) + { + return ToUpperAsciiInvariant(c); + } + + return Invariant.ChangeCase(c, toUpper: true); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void ChangeCaseToLower(ReadOnlySpan source, Span destination) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 7ac9c2afe6960..51de971360738 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -69,9 +69,7 @@ public readonly int IndexOfAny(ReadOnlySpan< { if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) { - return GlobalizationMode.UseNls - ? IndexOfAnyCaseInsensitiveUnicodeNls(span) - : IndexOfAnyCaseInsensitiveUnicodeIcuOrInvariant(span); + return IndexOfAnyCaseInsensitiveUnicode(span); } return IndexOfAnyCore(span); @@ -81,10 +79,7 @@ private readonly int IndexOfAnyCore(ReadOnly where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity where TFastScanVariant : struct, IFastScan { - if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) - { - throw new UnreachableException(); - } + Debug.Assert(typeof(TCaseSensitivity) != typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)); ref AhoCorasickNode nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); int nodeIndex = 0; @@ -171,11 +166,9 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), } // Mostly a copy of IndexOfAnyCore, but we may read two characters at a time in the case of surrogate pairs. - private readonly int IndexOfAnyCaseInsensitiveUnicodeIcuOrInvariant(ReadOnlySpan span) + private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnlySpan span) where TFastScanVariant : struct, IFastScan { - Debug.Assert(!GlobalizationMode.UseNls); - const char LowSurrogateNotSet = '\0'; ref AhoCorasickNode nodes = ref MemoryMarshal.GetArrayDataReference(_nodes); @@ -234,14 +227,22 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), (uint)(i + 1) < (uint)span.Length && char.IsLowSurrogate(lowSurrogate = Unsafe.Add(ref MemoryMarshal.GetReference(span), i + 1))) { - SurrogateCasing.ToUpper(c, lowSurrogate, out c, out lowSurrogateUpper); + if (GlobalizationMode.UseNls) + { + SurrogateToUpperNLS(c, lowSurrogate, out c, out lowSurrogateUpper); + } + else + { + SurrogateCasing.ToUpper(c, lowSurrogate, out c, out lowSurrogateUpper); + } + Debug.Assert(lowSurrogateUpper != LowSurrogateNotSet); } else { - c = GlobalizationMode.Invariant - ? InvariantModeCasing.ToUpper(c) - : OrdinalCasing.ToUpper(c); + c = GlobalizationMode.Invariant ? InvariantModeCasing.ToUpper(c) : + GlobalizationMode.UseNls ? TextInfo.ToUpperOrdinalNls(c) : + OrdinalCasing.ToUpper(c); } #if DEBUG @@ -297,78 +298,19 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), return result; } - private readonly int IndexOfAnyCaseInsensitiveUnicodeNls(ReadOnlySpan span) - where TFastScanVariant : struct, IFastScan + private static void SurrogateToUpperNLS(char h, char l, out char hr, out char lr) { - Debug.Assert(GlobalizationMode.UseNls); + Debug.Assert(char.IsHighSurrogate(h)); + Debug.Assert(char.IsLowSurrogate(l)); - if (span.IsEmpty) - { - return -1; - } + Span chars = stackalloc char[] { h, l }; + Span destination = stackalloc char[2]; - // If the input is large, we avoid uppercasing all of it upfront. - // We may find a match at position 0, so we want to behave closer to O(match offset) than O(input length). -#if DEBUG - // Make it easier to test with shorter inputs - const int StackallocThreshold = 32; -#else - // This limit isn't just about how much we allocate on the stack, but also how we chunk the input span. - // A larger value would improve throughput for rare matches, while a lower number reduces the overhead - // when matches are found close to the start. - const int StackallocThreshold = 64; -#endif + int written = Ordinal.ToUpperOrdinal(chars, destination); + Debug.Assert(written == 2); - int minBufferSize = (int)Math.Clamp(_maxValueLength * 4L, StackallocThreshold, string.MaxLength + 1); - - char[]? pooledArray = null; - Span buffer = minBufferSize <= StackallocThreshold - ? stackalloc char[StackallocThreshold] - : (pooledArray = ArrayPool.Shared.Rent(minBufferSize)); - - int leftoverFromPreviousIteration = 0; - int offsetFromStart = 0; - int result; - - while (true) - { - Span newSpaceAvailable = buffer.Slice(leftoverFromPreviousIteration); - int toConvert = Math.Min(span.Length, newSpaceAvailable.Length); - - int charsWritten = Ordinal.ToUpperOrdinal(span.Slice(0, toConvert), newSpaceAvailable); - Debug.Assert(charsWritten == toConvert); - span = span.Slice(toConvert); - - Span upperCaseBuffer = buffer.Slice(0, leftoverFromPreviousIteration + toConvert); - - // CaseSensitive instead of CaseInsensitiveUnicode as we've already done the case conversion. - result = IndexOfAnyCore(upperCaseBuffer); - - // Even if we found a result, it is possible that an earlier match exists if we ran out of upperCaseBuffer. - // If that is the case, we will find the correct result in the next loop iteration. - if (result >= 0 && (span.IsEmpty || result <= buffer.Length - _maxValueLength)) - { - result += offsetFromStart; - break; - } - - if (span.IsEmpty) - { - result = -1; - break; - } - - leftoverFromPreviousIteration = _maxValueLength - 1; - buffer.Slice(buffer.Length - leftoverFromPreviousIteration).CopyTo(buffer); - offsetFromStart += buffer.Length - leftoverFromPreviousIteration; - } - - if (pooledArray is not null) - { - ArrayPool.Shared.Return(pooledArray); - } - - return result; + hr = destination[0]; + lr = destination[1]; } public interface IFastScan { } From 7dc3186fc3d7e2cc5577fc2008f9efcaf9372e4a Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 21:46:17 +0200 Subject: [PATCH 06/43] Remove 2 TODOs --- .../src/System/SearchValues/Strings/StringSearchValues.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 7c42277f011de..ea15fde4fc756 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -117,8 +117,6 @@ private static SearchValues CreateFromNormalizedValues( return CreateForSingleValue(values[0], uniqueValues, ignoreCase, allAscii, asciiLettersOnly); } - // TODO: Should this be supported anywhere else? - // It may be too niche and too much code for WASM, but AOT with just Vector128 may be interesting. if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && TryGetTeddyAcceleratedValues(values, uniqueValues, ignoreCase, allAscii, asciiLettersOnly, nonAsciiAffectedByCaseConversion, minLength) is { } searchValues) { @@ -269,8 +267,6 @@ private static SearchValues PickTeddyImplementation Date: Wed, 5 Jul 2023 22:03:59 +0200 Subject: [PATCH 07/43] Add a few more simple test cases --- .../System.Memory/tests/Span/StringSearchValues.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index 0b5d195d96363..50618b371f66a 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -109,6 +109,16 @@ void AssertIndexOfAnyAndFriends(Span values, int any, int anyExcept, int [InlineData(StringComparison.OrdinalIgnoreCase, -1, " ", null)] [InlineData(StringComparison.Ordinal, -1, "", null)] [InlineData(StringComparison.OrdinalIgnoreCase, -1, "", null)] + // A few simple cases + [InlineData(StringComparison.Ordinal, 1, "xbc", "abc, bc")] + [InlineData(StringComparison.Ordinal, 0, "foobar", "foo, bar")] + [InlineData(StringComparison.Ordinal, 0, "barfoo", "foo, bar")] + [InlineData(StringComparison.Ordinal, 0, "foofoo", "foo, bar")] + [InlineData(StringComparison.Ordinal, 0, "barbar", "foo, bar")] + [InlineData(StringComparison.Ordinal, 4, "bafofoo", "foo, bar")] + [InlineData(StringComparison.Ordinal, 4, "bafofoo", "bar, foo")] + [InlineData(StringComparison.Ordinal, 4, "fobabar", "foo, bar")] + [InlineData(StringComparison.Ordinal, 4, "fobabar", "bar, foo")] // Multiple potential matches - we want the first one [InlineData(StringComparison.Ordinal, 1, "abcd", "bc, cd")] // Simple case sensitivity From b412fd995eb713096a302d55e9c654f4f64236ec Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 22:13:09 +0200 Subject: [PATCH 08/43] Add BurntSushi/aho-corasick to THIRD-PARTY-NOTICES --- THIRD-PARTY-NOTICES.TXT | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 53855810b5af9..7026e2bc232f7 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -1270,3 +1270,30 @@ Licensed under the Apache License, Version 2.0. Available at https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/LICENSE + +License for the Teddy multi-substring searching implementation +-------------------------------------- + +https://github.com/BurntSushi/aho-corasick + +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. From 7f8bae21b815a51be507f8b75e9d94b779820de4 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 22:24:22 +0200 Subject: [PATCH 09/43] Add more rng to stress runs --- .../tests/Span/StringSearchValues.cs | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index 50618b371f66a..15dedc9b8deb5 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -239,7 +239,8 @@ static void RunStress() { var helper = new StringSearchValuesTestHelper( expected: IndexOfAnyReferenceImpl, - searchValues: (searchSpace, values) => searchSpace.IndexOfAny(values)) + searchValues: (searchSpace, values) => searchSpace.IndexOfAny(values), + rngSeed: Random.Shared.Next()) { MaxNeedleCount = maxNeedleCount, MaxNeedleValueLength = maxNeedleValueLength, @@ -294,34 +295,44 @@ private static void RunUsingNLS(Action action) private sealed class StringSearchValuesTestHelper { + public delegate int IndexOfAnySearchDelegate(ReadOnlySpan searchSpace, ReadOnlySpan values, StringComparison comparisonType); + + public delegate int SearchValuesSearchDelegate(ReadOnlySpan searchSpace, SearchValues values); + public int MaxNeedleCount = 20; public int MaxNeedleValueLength = 10; public int MaxHaystackLength = 100; public int HaystackIterationsPerNeedle = 50; public int MinValueLength = 1; - private static readonly char[] s_randomAsciiChars; - private static readonly char[] s_randomSimpleAsciiChars; - private static readonly char[] s_randomChars; + private readonly IndexOfAnySearchDelegate _expectedDelegate; + private readonly SearchValuesSearchDelegate _searchValuesDelegate; - static StringSearchValuesTestHelper() + private readonly char[] _randomAsciiChars; + private readonly char[] _randomSimpleAsciiChars; + private readonly char[] _randomChars; + + public StringSearchValuesTestHelper(IndexOfAnySearchDelegate expected, SearchValuesSearchDelegate searchValues, int rngSeed = 42) { - s_randomAsciiChars = new char[100 * 1024]; - s_randomSimpleAsciiChars = new char[100 * 1024]; - s_randomChars = new char[1024 * 1024]; + _expectedDelegate = expected; + _searchValuesDelegate = searchValues; - var rng = new Random(42); + _randomAsciiChars = new char[100 * 1024]; + _randomSimpleAsciiChars = new char[100 * 1024]; + _randomChars = new char[1024 * 1024]; - for (int i = 0; i < s_randomAsciiChars.Length; i++) + var rng = new Random(rngSeed); + + for (int i = 0; i < _randomAsciiChars.Length; i++) { - s_randomAsciiChars[i] = (char)rng.Next(0, 128); + _randomAsciiChars[i] = (char)rng.Next(0, 128); } - for (int i = 0; i < s_randomSimpleAsciiChars.Length; i++) + for (int i = 0; i < _randomSimpleAsciiChars.Length; i++) { int random = rng.Next(26 * 2 + 10); - s_randomSimpleAsciiChars[i] = (char)(random + (random switch + _randomSimpleAsciiChars[i] = (char)(random + (random switch { < 10 => '0', < 36 => 'a' - 10, @@ -329,22 +340,9 @@ static StringSearchValuesTestHelper() })); } - rng.NextBytes(MemoryMarshal.Cast(s_randomChars)); - } - - private readonly IndexOfAnySearchDelegate _expectedDelegate; - private readonly SearchValuesSearchDelegate _searchValuesDelegate; - - public StringSearchValuesTestHelper(IndexOfAnySearchDelegate expected, SearchValuesSearchDelegate searchValues) - { - _expectedDelegate = expected; - _searchValuesDelegate = searchValues; + rng.NextBytes(MemoryMarshal.Cast(_randomChars)); } - public delegate int IndexOfAnySearchDelegate(ReadOnlySpan searchSpace, ReadOnlySpan values, StringComparison comparisonType); - - public delegate int SearchValuesSearchDelegate(ReadOnlySpan searchSpace, SearchValues values); - public void StressRandomInputs(TimeSpan duration) { ExceptionDispatchInfo? exception = null; @@ -375,14 +373,14 @@ public void TestRandomInputs(int iterationCount = 1_000, Random? rng = null) for (int iterations = 0; iterations < iterationCount; iterations++) { // There are more interesting corner cases with ASCII needles, test those more. - Test(rng, s_randomSimpleAsciiChars, s_randomSimpleAsciiChars); - Test(rng, s_randomAsciiChars, s_randomSimpleAsciiChars); - Test(rng, s_randomSimpleAsciiChars, s_randomAsciiChars); - Test(rng, s_randomAsciiChars, s_randomAsciiChars); - Test(rng, s_randomChars, s_randomSimpleAsciiChars); - Test(rng, s_randomChars, s_randomAsciiChars); - - Test(rng, s_randomChars, s_randomChars); + Test(rng, _randomSimpleAsciiChars, _randomSimpleAsciiChars); + Test(rng, _randomAsciiChars, _randomSimpleAsciiChars); + Test(rng, _randomSimpleAsciiChars, _randomAsciiChars); + Test(rng, _randomAsciiChars, _randomAsciiChars); + Test(rng, _randomChars, _randomSimpleAsciiChars); + Test(rng, _randomChars, _randomAsciiChars); + + Test(rng, _randomChars, _randomChars); } } From c5584042c79a28b4fc9c8a992d1728b0a5a3f92d Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 22:33:20 +0200 Subject: [PATCH 10/43] Remove DebuggerDisplay on AhoCorasickNode --- .../Strings/Helpers/AhoCorasickNode.cs | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs index 55746cab54654..02de1be5fc603 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs @@ -8,7 +8,6 @@ namespace System.Buffers { - [DebuggerDisplay("{DebuggerDisplay,nq}")] internal struct AhoCorasickNode { private static object EmptyChildrenSentinel => Array.Empty(); @@ -172,30 +171,5 @@ static int DictionarySizeEstimate(int childCount) } } } - - private readonly string DebuggerDisplay - { - get - { - int count = _firstChildChar >= 0 ? 1 : 0; - - if (_children is int[] table) - { - foreach (int index in table) - { - if (index >= 0) - { - count++; - } - } - } - else - { - count += ((Dictionary)_children).Count; - } - - return $"MatchLength={MatchLength} SuffixLink={SuffixLink} ChildrenCount={count}"; - } - } } } From 15996f45cd10a24d8772c10b630978d9d685df21 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 5 Jul 2023 22:36:49 +0200 Subject: [PATCH 11/43] More comments around AhoCorasickNode.TryCreateJumpTable --- .../Strings/Helpers/AhoCorasickNode.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs index 02de1be5fc603..512b30fcf37ec 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs @@ -134,8 +134,8 @@ static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true maxValue = Math.Max(maxValue, childChar); } - int tableSize = TableSizeEstimate(maxValue); - int dictionarySize = DictionarySizeEstimate(children.Count); + int tableSize = TableBytesEstimate(maxValue); + int dictionarySize = DictionaryBytesEstimate(children.Count); if (tableSize > dictionarySize * AcceptableSizeMultiplier) { @@ -154,13 +154,19 @@ static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true return true; - static int TableSizeEstimate(int maxValue) + static int TableBytesEstimate(int maxValue) { - return 32 + (maxValue * 4); + // An approximate number of bytes consumed by an + // int[] table with a known number of entries. + // Only used as a heuristic, so numbers don't have to be exact. + return 32 + (maxValue * sizeof(int)); } - static int DictionarySizeEstimate(int childCount) + static int DictionaryBytesEstimate(int childCount) { + // An approximate number of bytes consumed by a + // Dictionary with a known number of entries. + // Only used as a heuristic, so numbers don't have to be exact. return childCount switch { < 4 => 192, From 5d4cacc52896f4de949f545b020e6c0e1f967c0b Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 8 Jul 2023 03:23:44 +0200 Subject: [PATCH 12/43] Reduce RabinKarp ctor allocations --- .../Strings/Helpers/AhoCorasick.cs | 9 ++--- .../SearchValues/Strings/Helpers/RabinKarp.cs | 36 +++++++++---------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 51de971360738..35c4be3e49960 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -67,12 +67,9 @@ public readonly int IndexOfAny(ReadOnlySpan< where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity where TFastScanVariant : struct, IFastScan { - if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) - { - return IndexOfAnyCaseInsensitiveUnicode(span); - } - - return IndexOfAnyCore(span); + return typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode) + ? IndexOfAnyCaseInsensitiveUnicode(span) + : IndexOfAnyCore(span); } private readonly int IndexOfAnyCore(ReadOnlySpan span) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index 3a707ac619812..ef93ea07f8951 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Runtime.CompilerServices; @@ -51,7 +50,7 @@ public RabinKarp(ReadOnlySpan values) _hashLength = minimumLength; _hashUpdateMultiplier = (nuint)1 << ((minimumLength - 1) * HashShiftPerElement); - var bucketLists = new List?[BucketCount]; + string[][] buckets = _buckets = new string[BucketCount][]; foreach (string value in values) { @@ -62,37 +61,36 @@ public RabinKarp(ReadOnlySpan values) } nuint bucket = hash % BucketCount; - var bucketList = bucketLists[bucket] ??= new List(); - bucketList.Add(value); - } + string[] newBucket; - var buckets = new string[BucketCount][]; - for (int i = 0; i < bucketLists.Length; i++) - { - if (bucketLists[i] is List list) + if (buckets[bucket] is string[] existingBucket) { - buckets[i] = list.ToArray(); + newBucket = new string[existingBucket.Length + 1]; + existingBucket.AsSpan().CopyTo(newBucket); + } + else + { + newBucket = new string[1]; } - } - _buckets = buckets; + newBucket[^1] = value; + buckets[bucket] = newBucket; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public readonly int IndexOfAny(ReadOnlySpan span) - where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity => - typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode) + where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + { + return typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode) ? IndexOfAnyCaseInsensitiveUnicode(span) : IndexOfAnyCore(span); + } private readonly int IndexOfAnyCore(ReadOnlySpan span) where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity { - if (typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)) - { - throw new UnreachableException(); - } - + Debug.Assert(typeof(TCaseSensitivity) != typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)); Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled short inputs."); ref char current = ref MemoryMarshal.GetReference(span); From b8757e4d3cb17cf7190393c636871a6b483ab8df Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 8 Jul 2023 03:32:10 +0200 Subject: [PATCH 13/43] Move ordinal helpers around a bit --- .../src/System/Globalization/TextInfo.cs | 16 ++++++++++++---- .../SearchValues/Strings/Helpers/AhoCorasick.cs | 4 +--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs index 172778324166d..a035c875f7290 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TextInfo.cs @@ -190,14 +190,22 @@ private unsafe char ChangeCase(char c, bool toUpper) return dst; } - internal static char ToUpperOrdinalNls(char c) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static char ToUpperOrdinal(char c) { - if (char.IsAscii(c)) + if (GlobalizationMode.Invariant) { - return ToUpperAsciiInvariant(c); + return InvariantModeCasing.ToUpper(c); } - return Invariant.ChangeCase(c, toUpper: true); + if (GlobalizationMode.UseNls) + { + return char.IsAscii(c) + ? ToUpperAsciiInvariant(c) + : Invariant.ChangeCase(c, toUpper: true); + } + + return OrdinalCasing.ToUpper(c); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 35c4be3e49960..10102830350d0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -237,9 +237,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), } else { - c = GlobalizationMode.Invariant ? InvariantModeCasing.ToUpper(c) : - GlobalizationMode.UseNls ? TextInfo.ToUpperOrdinalNls(c) : - OrdinalCasing.ToUpper(c); + c = TextInfo.ToUpperOrdinal(c); } #if DEBUG From 822f5c6e69570255cb148282708bdda59b87acc7 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 8 Jul 2023 03:36:43 +0200 Subject: [PATCH 14/43] Typo --- .../src/System/SearchValues/Strings/Helpers/AhoCorasick.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 10102830350d0..f49facb0afdc1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -11,7 +11,7 @@ namespace System.Buffers { /// /// An implementation of the Aho-Corasick algorithm we use as a fallback when we can't use Teddy - /// (either due to missing hardware intrinsics, or due to characteristics of the of values used). + /// (either due to missing hardware intrinsics, or due to characteristics of the values used). /// https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm /// internal readonly struct AhoCorasick From 2e70415c421151461205ba04f075db62a1232bdf Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 8 Jul 2023 04:00:55 +0200 Subject: [PATCH 15/43] Remove IValueLength --- .../Helpers/StringSearchValuesHelper.cs | 113 ------------------ .../SingleStringSearchValuesThreeChars.cs | 11 +- .../Strings/StringSearchValues.cs | 55 +++------ 3 files changed, 20 insertions(+), 159 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index 8c38d7425db40..1b321a1f25b48 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -5,7 +5,6 @@ using System.Globalization; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; -using System.Text; namespace System.Buffers { @@ -38,30 +37,6 @@ public static bool StartsWith(ref char matchStart, int lengthR return TCaseSensitivity.Equals(ref matchStart, candidate); } - public interface IValueLength - { - static abstract bool AtLeast4Chars { get; } - static abstract bool AtLeast8Chars { get; } - } - - public readonly struct ValueLengthLessThan4 : IValueLength - { - public static bool AtLeast4Chars => false; - public static bool AtLeast8Chars => false; - } - - public readonly struct ValueLength4To7 : IValueLength - { - public static bool AtLeast4Chars => true; - public static bool AtLeast8Chars => false; - } - - public readonly struct ValueLength8OrLonger : IValueLength - { - public static bool AtLeast4Chars => true; - public static bool AtLeast8Chars => true; - } - public interface ICaseSensitivity { static abstract char TransformInput(char input); @@ -69,7 +44,6 @@ public interface ICaseSensitivity static abstract Vector256 TransformInput(Vector256 input); static abstract Vector512 TransformInput(Vector512 input); static abstract bool Equals(ref char matchStart, string candidate); - static abstract bool Equals(ref char matchStart, string candidate) where TValueLength : struct, IValueLength; } public readonly struct CaseSensitive : ICaseSensitivity @@ -106,37 +80,6 @@ public static bool Equals(ref char matchStart, string candidate) return true; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - where TValueLength : struct, IValueLength - { - Debug.Assert(candidate.Length > 1); - Debug.Assert(matchStart == candidate[0], "This should only be called after the first character has been checked"); - - ref byte first = ref Unsafe.As(ref matchStart); - ref byte second = ref Unsafe.As(ref candidate.GetRawStringData()); - nuint byteLength = (nuint)(uint)candidate.Length * 2; - - if (TValueLength.AtLeast8Chars) - { - return SpanHelpers.SequenceEqual(ref first, ref second, byteLength); - } - else if (TValueLength.AtLeast4Chars) - { - nuint offset = byteLength - sizeof(ulong); - ulong differentBits = Unsafe.ReadUnaligned(ref first) - Unsafe.ReadUnaligned(ref second); - differentBits |= Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); - return differentBits == 0; - } - else - { - nuint offset = byteLength - sizeof(uint); - - return Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) - == Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); - } - } } public readonly struct CaseInsensitiveAsciiLetters : ICaseSensitivity @@ -166,38 +109,6 @@ public static bool Equals(ref char matchStart, string candidate) return true; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - where TValueLength : struct, IValueLength - { - Debug.Assert(candidate.Length > 1); - Debug.Assert(candidate.ToUpperInvariant() == candidate); - - if (TValueLength.AtLeast8Chars) - { - return Ascii.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), (uint)candidate.Length); - } - - ref byte first = ref Unsafe.As(ref matchStart); - ref byte second = ref Unsafe.As(ref candidate.GetRawStringData()); - nuint byteLength = (nuint)(uint)candidate.Length * 2; - - if (TValueLength.AtLeast4Chars) - { - nuint offset = byteLength - sizeof(ulong); - ulong differentBits = (Unsafe.ReadUnaligned(ref first) & ~0x20002000200020u) - Unsafe.ReadUnaligned(ref second); - differentBits |= (Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) & ~0x20002000200020u) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); - return differentBits == 0; - } - else - { - nuint offset = byteLength - sizeof(uint); - uint differentBits = (Unsafe.ReadUnaligned(ref first) & ~0x200020u) - Unsafe.ReadUnaligned(ref second); - differentBits |= (Unsafe.ReadUnaligned(ref Unsafe.Add(ref first, offset)) & ~0x200020u) - Unsafe.ReadUnaligned(ref Unsafe.Add(ref second, offset)); - return differentBits == 0; - } - } } public readonly struct CaseInsensitiveAscii : ICaseSensitivity @@ -251,18 +162,6 @@ public static bool Equals(ref char matchStart, string candidate) return true; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - where TValueLength : struct, IValueLength - { - if (Vector128.IsHardwareAccelerated && TValueLength.AtLeast8Chars) - { - return Ascii.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), (uint)candidate.Length); - } - - return Equals(ref matchStart, candidate); - } } public readonly struct CaseInsensitiveUnicode : ICaseSensitivity @@ -277,18 +176,6 @@ public static bool Equals(ref char matchStart, string candidate) { return Ordinal.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - where TValueLength : struct, IValueLength - { - if (Vector128.IsHardwareAccelerated && TValueLength.AtLeast8Chars) - { - return Ordinal.EqualsIgnoreCase_Vector128(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); - } - - return Ordinal.EqualsIgnoreCase_Scalar(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); - } } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index 72b900ac7d31e..98d4b0c481bf1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -13,8 +13,7 @@ namespace System.Buffers { // Based on SpanHelpers.IndexOf(ref char, int, ref char, int) // This implementation uses 3 precomputed anchor points when searching. - internal sealed class SingleStringSearchValuesThreeChars : StringSearchValuesBase - where TValueLength : struct, IValueLength + internal sealed class SingleStringSearchValuesThreeChars : StringSearchValuesBase where TCaseSensitivity : struct, ICaseSensitivity { private const ushort CaseConversionMask = unchecked((ushort)~0x20); @@ -203,7 +202,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) ref char cur = ref Unsafe.Add(ref searchSpace, i); if ((typeof(TCaseSensitivity) == typeof(CaseInsensitiveUnicode) || TCaseSensitivity.TransformInput(cur) == valueHead) && - TCaseSensitivity.Equals(ref cur, value)) + TCaseSensitivity.Equals(ref cur, value)) { return (int)i; } @@ -285,8 +284,7 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); - if ((typeof(TCaseSensitivity) == typeof(CaseSensitive) && !TValueLength.AtLeast4Chars) || - TCaseSensitivity.Equals(ref matchRef, _value)) + if (TCaseSensitivity.Equals(ref matchRef, _value)) { offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); return true; @@ -310,8 +308,7 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mas ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); - if ((typeof(TCaseSensitivity) == typeof(CaseSensitive) && !TValueLength.AtLeast4Chars) || - TCaseSensitivity.Equals(ref matchRef, _value)) + if (TCaseSensitivity.Equals(ref matchRef, _value)) { offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); return true; diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index ea15fde4fc756..c26f0b26333aa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -294,16 +294,25 @@ private static SearchValues CreateForSingleValue( if (Vector128.IsHardwareAccelerated && value.Length > 1 && value.Length <= maxLength) { - SearchValues? searchValues = value.Length switch + if (!ignoreCase) { - < 4 => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), - < 8 => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), - _ => TryCreateSingleValuesThreeChars(value, uniqueValues, ignoreCase, allAscii, asciiLettersOnly), - }; + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + if (asciiLettersOnly) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } - if (searchValues is not null) + if (allAscii) { - return searchValues; + return new SingleStringSearchValuesThreeChars(value, uniqueValues); + } + + // When ignoring casing, all anchor chars we search for must be ASCII. + if (char.IsAscii(value[0]) && value.AsSpan().LastIndexOfAnyInRange((char)0, (char)127) > 0) + { + return new SingleStringSearchValuesThreeChars(value, uniqueValues); } } @@ -312,38 +321,6 @@ private static SearchValues CreateForSingleValue( : new SingleStringSearchValuesFallback(value, uniqueValues); } - private static SearchValues? TryCreateSingleValuesThreeChars( - string value, - HashSet uniqueValues, - bool ignoreCase, - bool allAscii, - bool asciiLettersOnly) - where TValueLength : struct, IValueLength - { - if (!ignoreCase) - { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); - } - - if (asciiLettersOnly) - { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); - } - - if (allAscii) - { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); - } - - // When ignoring casing, all anchor chars we search for must be ASCII. - if (char.IsAscii(value[0]) && value.AsSpan().LastIndexOfAnyInRange((char)0, (char)127) > 0) - { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); - } - - return null; - } - private static void AnalyzeValues( ReadOnlySpan values, ref bool ignoreCase, From 676c461efcff96a8c7781e2f9a7257a057b946c1 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 10 Jul 2023 00:26:48 +0200 Subject: [PATCH 16/43] Dedup some ICaseSensitivity.Equals loops --- .../Helpers/StringSearchValuesHelper.cs | 70 ++++++------------- 1 file changed, 23 insertions(+), 47 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index 1b321a1f25b48..f28492dc452f2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -37,6 +37,21 @@ public static bool StartsWith(ref char matchStart, int lengthR return TCaseSensitivity.Equals(ref matchStart, candidate); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ScalarEquals(ref char matchStart, string candidate) + where TCaseSensitivity : struct, ICaseSensitivity + { + for (int i = 0; i < candidate.Length; i++) + { + if (TCaseSensitivity.TransformInput(candidate[i]) != Unsafe.Add(ref matchStart, i)) + { + return false; + } + } + + return true; + } + public interface ICaseSensitivity { static abstract char TransformInput(char input); @@ -61,25 +76,8 @@ public interface ICaseSensitivity public static Vector512 TransformInput(Vector512 input) => input; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - { - ref char end = ref Unsafe.Add(ref matchStart, candidate.Length); - ref char candidateRef = ref Unsafe.AsRef(candidate.GetPinnableReference()); - - do - { - if (candidateRef != matchStart) - { - return false; - } - - matchStart = ref Unsafe.Add(ref matchStart, 1); - candidateRef = ref Unsafe.Add(ref candidateRef, 1); - } - while (Unsafe.IsAddressLessThan(ref matchStart, ref end)); - - return true; - } + public static bool Equals(ref char matchStart, string candidate) => + ScalarEquals(ref matchStart, candidate); } public readonly struct CaseInsensitiveAsciiLetters : ICaseSensitivity @@ -97,18 +95,8 @@ public static bool Equals(ref char matchStart, string candidate) public static Vector512 TransformInput(Vector512 input) => input & Vector512.Create(unchecked((byte)~0x20)); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - { - for (int i = 0; i < candidate.Length; i++) - { - if ((Unsafe.Add(ref matchStart, i) & ~0x20) != candidate[i]) - { - return false; - } - } - - return true; - } + public static bool Equals(ref char matchStart, string candidate) => + ScalarEquals(ref matchStart, candidate); } public readonly struct CaseInsensitiveAscii : ICaseSensitivity @@ -150,18 +138,8 @@ public static Vector512 TransformInput(Vector512 input) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - { - for (int i = 0; i < candidate.Length; i++) - { - if (TextInfo.ToUpperAsciiInvariant(Unsafe.Add(ref matchStart, i)) != candidate[i]) - { - return false; - } - } - - return true; - } + public static bool Equals(ref char matchStart, string candidate) => + ScalarEquals(ref matchStart, candidate); } public readonly struct CaseInsensitiveUnicode : ICaseSensitivity @@ -172,10 +150,8 @@ public static bool Equals(ref char matchStart, string candidate) public static Vector512 TransformInput(Vector512 input) => throw new UnreachableException(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool Equals(ref char matchStart, string candidate) - { - return Ordinal.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); - } + public static bool Equals(ref char matchStart, string candidate) => + Ordinal.EqualsIgnoreCase(ref matchStart, ref candidate.GetRawStringData(), candidate.Length); } } } From 6e31d6e42462873d535ebfcc569e14c74c9942be Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 10 Jul 2023 00:40:23 +0200 Subject: [PATCH 17/43] Avoid some costs in RabinKarp for long values --- .../SearchValues/Strings/Helpers/RabinKarp.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index ef93ea07f8951..274285e130965 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -50,6 +50,14 @@ public RabinKarp(ReadOnlySpan values) _hashLength = minimumLength; _hashUpdateMultiplier = (nuint)1 << ((minimumLength - 1) * HashShiftPerElement); + if (minimumLength > MaxInputLength) + { + // All the values are long. They'll either be handled by Teddy or won't match at all. + // There's no point in allocating the buckets as they will never be accessed. + _buckets = null!; + return; + } + string[][] buckets = _buckets = new string[BucketCount][]; foreach (string value in values) @@ -142,6 +150,12 @@ private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnlySpan span) { Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled long inputs."); + if (_hashLength > span.Length) + { + // Can't possibly match, all the values are longer than our input span. + return -1; + } + Span upperCase = stackalloc char[MaxInputLength].Slice(0, span.Length); int charsWritten = Ordinal.ToUpperOrdinal(span, upperCase); From 38a643697a8fe82d3f6efad17a987e0f4a0e571d Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 10 Jul 2023 01:17:11 +0200 Subject: [PATCH 18/43] Remove now-unused maxValueLength field --- .../src/System/SearchValues/Strings/Helpers/AhoCorasick.cs | 4 +--- .../System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index f49facb0afdc1..f7a19a013b831 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -18,13 +18,11 @@ internal readonly struct AhoCorasick { private readonly AhoCorasickNode[] _nodes; private readonly Vector256 _startingCharsAsciiBitmap; - private readonly int _maxValueLength; // Only used by the NLS fallback - public AhoCorasick(AhoCorasickNode[] nodes, Vector256 startingAsciiBitmap, int maxValueLength) + public AhoCorasick(AhoCorasickNode[] nodes, Vector256 startingAsciiBitmap) { _nodes = nodes; _startingCharsAsciiBitmap = startingAsciiBitmap; - _maxValueLength = maxValueLength; } public readonly bool ShouldUseAsciiFastScan diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs index d625f12d70e74..7449c40855699 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs @@ -18,7 +18,6 @@ internal ref struct AhoCorasickBuilder private ValueListBuilder _nodes; private ValueListBuilder _parents; private Vector256 _startingCharsAsciiBitmap; - private int _maxValueLength; // Only used by the NLS fallback public AhoCorasickBuilder(ReadOnlySpan values, bool ignoreCase, ref HashSet? unreachableValues) { @@ -54,7 +53,7 @@ public AhoCorasick Build() GenerateStartingAsciiCharsBitmap(); } - return new AhoCorasick(_nodes.AsSpan().ToArray(), _startingCharsAsciiBitmap, _maxValueLength); + return new AhoCorasick(_nodes.AsSpan().ToArray(), _startingCharsAsciiBitmap); } public void Dispose() @@ -101,7 +100,6 @@ private void BuildTrie(ref HashSet? unreachableValues) if (i == value.Length - 1) { node.MatchLength = value.Length; - _maxValueLength = Math.Max(_maxValueLength, value.Length); break; } } From 2f7a96638d86d0817f7bdd2757c301fe0bc7345d Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 10 Jul 2023 03:36:24 +0200 Subject: [PATCH 19/43] Add comment about MaxCombinedFrequency in AC --- .../System/SearchValues/Strings/Helpers/AhoCorasick.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index f7a19a013b831..9a747c73ba011 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -36,11 +36,14 @@ public readonly bool ShouldUseAsciiFastScan // If there are a lot of starting characters such that we often find one early, // the ASCII fast scan may end up performing worse than checking one character at a time. // Avoid using this optimization if the combined frequency of starting chars is too high. - - // Combined frequency of characters based on CharacterFrequencyHelper.AsciiFrequency: + // + // For reference, the combined frequency of characters based on CharacterFrequencyHelper.AsciiFrequency: // - All digits is ~ 5 % // - All lowercase letters is ~ 57.2 % // - All uppercase letters is ~ 7.4 % + // + // This limit is based on experimentation with different texts and sets of values. + // Above ~50 %, the cost of calling into the vectorized helper is higher than checking char by char on average. const float MaxCombinedFrequency = 50f; float frequency = 0; @@ -53,7 +56,7 @@ public readonly bool ShouldUseAsciiFastScan } } - return frequency < MaxCombinedFrequency; + return frequency <= MaxCombinedFrequency; } return false; From 06851729c968a83c2066734b56cd376e0dba7d66 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 19 Jul 2023 16:54:38 +0200 Subject: [PATCH 20/43] React to compiler changes to InlineArray --- .../AsciiStringSearchValuesTeddyBase.cs | 9 ++++--- .../Strings/Helpers/EightPackedReferences.cs | 25 +++---------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index a5b1c8113f256..b39ff28f74302 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -438,7 +438,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector1 { int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); - object bucket = _buckets[candidateOffset]; + object? bucket = _buckets[candidateOffset]; + Debug.Assert(bucket is not null); if (TBucketized.Value ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) @@ -478,7 +479,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector2 { int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); - object bucket = _buckets[candidateOffset]; + object? bucket = _buckets[candidateOffset]; + Debug.Assert(bucket is not null); if (TBucketized.Value ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) @@ -518,7 +520,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector5 { int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); - object bucket = _buckets[candidateOffset]; + object? bucket = _buckets[candidateOffset]; + Debug.Assert(bucket is not null); if (TBucketized.Value ? StartsWith(ref matchRef, lengthRemaining, Unsafe.As(bucket)) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs index 43192751a9d70..b9c1a85e5e913 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs @@ -9,32 +9,15 @@ namespace System.Buffers [InlineArray(8)] internal readonly struct EightPackedReferences { +#pragma warning disable CA1823 // Unused field -- TODO: Why is this needed? private readonly object? _ref0; +#pragma warning restore CA1823 public EightPackedReferences(ReadOnlySpan values) { - Debug.Assert(values.Length <= 8, $"Got {values.Length} values"); + Debug.Assert(values.Length is > 0 and <= 8, $"Got {values.Length} values"); - for (int i = 0; i < values.Length; i++) - { - this[i] = values[i]; - } - } - - public object this[int index] - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get - { - Debug.Assert(index is >= 0 and < 8, $"Should be [0, 7], was {index}"); - Debug.Assert(Unsafe.Add(ref Unsafe.AsRef(in _ref0), index) is not null); - - return Unsafe.Add(ref Unsafe.AsRef(in _ref0), index)!; - } - private set - { - Unsafe.Add(ref Unsafe.AsRef(in _ref0), index) = value; - } + values.CopyTo(this!); } } } From 57f2c3b1ff445e34a925b9afd867b0b63e8cc455 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 21 Jul 2023 13:29:23 +0200 Subject: [PATCH 21/43] Link the roslyn-analyzers issue --- .../SearchValues/Strings/Helpers/EightPackedReferences.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs index b9c1a85e5e913..d667f628c7dc6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs @@ -9,7 +9,7 @@ namespace System.Buffers [InlineArray(8)] internal readonly struct EightPackedReferences { -#pragma warning disable CA1823 // Unused field -- TODO: Why is this needed? +#pragma warning disable CA1823 // Unused field -- https://github.com/dotnet/roslyn-analyzers/issues/6788 private readonly object? _ref0; #pragma warning restore CA1823 From 5f46a3a6629264e77a66532ee373e32d708191e3 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 22 Jul 2023 19:30:35 +0200 Subject: [PATCH 22/43] Remove >>> workaround now that #86841 is merged --- .../SearchValues/Strings/Helpers/TeddyHelper.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs index c2e190026c0c4..b7e84dac0980e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs @@ -196,10 +196,7 @@ private static (Vector128 Low, Vector128 High) GetNibbles(Vector128< ? input : input & Vector128.Create((byte)0xF); - // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 - Vector128 high = AdvSimd.IsSupported - ? AdvSimd.ShiftRightLogical(input, 4) - : (input.AsInt32() >>> 4).AsByte() & Vector128.Create((byte)0xF); + Vector128 high = input >>> 4; return (low, high); } @@ -211,8 +208,7 @@ private static (Vector256 Low, Vector256 High) GetNibbles(Vector256< // of doing an implicit 'AND 0xF' in order to skip the redundant AND. Vector256 low = input; - // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 - Vector256 high = (input.AsInt32() >>> 4).AsByte() & Vector256.Create((byte)0xF); + Vector256 high = input >>> 4; return (low, high); } @@ -224,8 +220,7 @@ private static (Vector512 Low, Vector512 High) GetNibbles(Vector512< // of doing an implicit 'AND 0xF' in order to skip the redundant AND. Vector512 low = input; - // X86 doesn't have a logical right shift intrinsic for bytes: https://github.com/dotnet/runtime/issues/82564 - Vector512 high = (input.AsInt32() >>> 4).AsByte() & Vector512.Create((byte)0xF); + Vector512 high = input >>> 4; return (low, high); } From 9ea478434cf737419b2e02668e2644db44edd5af Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 01:42:53 +0200 Subject: [PATCH 23/43] Improve comments around the Teddy implementation --- .../AsciiStringSearchValuesTeddyBase.cs | 43 ++++++++++ .../Strings/Helpers/TeddyBucketizer.cs | 8 ++ .../Strings/Helpers/TeddyHelper.cs | 81 ++++++++++++++++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index b39ff28f74302..f5c3c37f35b66 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -14,17 +14,25 @@ namespace System.Buffers { + // For a description of the Teddy multi-substring matching algorithm, + // see https://github.com/BurntSushi/aho-corasick/blob/master/src/packed/teddy/README.md internal abstract class AsciiStringSearchValuesTeddyBase : StringSearchValuesRabinKarp where TBucketized : struct, SearchValues.IRuntimeConst where TStartCaseSensitivity : struct, ICaseSensitivity where TCaseSensitivity : struct, ICaseSensitivity { + // We may be using N2 or N3 mode depending on whether we're checking 2 or 3 starting bytes for each bucket. + // The result of ProcessInputN2 and ProcessInputN3 are offset by 1 and 2 positions respectively (MatchStartOffsetN2 and MatchStartOffsetN3). + // See the full description of TeddyHelper.ProcessInputN3 for more details about why these constants exist. private const int MatchStartOffsetN2 = 1; private const int MatchStartOffsetN3 = 2; private const int CharsPerIterationVector128 = 16; private const int CharsPerIterationAvx2 = 32; private const int CharsPerIterationAvx512 = 64; + // We may have up to 8 buckets. + // If we have <= 8 strings, the buckets will be the strings themselves, and TBucketized.Value will be false. + // If we have more than 8, the buckets will be string[], and TBucketized.Value will be true. private readonly EightPackedReferences _buckets; private readonly Vector512 @@ -112,6 +120,8 @@ protected int IndexOfAnyN3(ReadOnlySpan span) [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] private int IndexOfAnyN2Vector128(ReadOnlySpan span) { + // See comments in 'IndexOfAnyN3Vector128' below. + // This method is the same, but compares 2 starting chars instead of 3. if (span.Length < CharsPerIterationVector128 + MatchStartOffsetN2) { return ShortInputFallback(span); @@ -164,6 +174,8 @@ private int IndexOfAnyN2Vector128(ReadOnlySpan span) [CompExactlyDependsOn(typeof(Avx2))] private int IndexOfAnyN2Avx2(ReadOnlySpan span) { + // See comments in 'IndexOfAnyN3Vector128' below. + // This method is the same, but operates on 32 input characters at a time and compares 2 starting chars instead of 3. Debug.Assert(span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN2); ref char searchSpace = ref MemoryMarshal.GetReference(span); @@ -213,6 +225,8 @@ private int IndexOfAnyN2Avx2(ReadOnlySpan span) [CompExactlyDependsOn(typeof(Avx512BW))] private int IndexOfAnyN2Avx512(ReadOnlySpan span) { + // See comments in 'IndexOfAnyN3Vector128' below. + // This method is the same, but operates on 64 input characters at a time and compares 2 starting chars instead of 3. Debug.Assert(span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN2); ref char searchSpace = ref MemoryMarshal.GetReference(span); @@ -263,6 +277,7 @@ private int IndexOfAnyN2Avx512(ReadOnlySpan span) [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] private int IndexOfAnyN3Vector128(ReadOnlySpan span) { + // We can't process inputs shorter than 18 characters in a vectorized manner here. if (span.Length < CharsPerIterationVector128 + MatchStartOffsetN3) { return ShortInputFallback(span); @@ -273,15 +288,27 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) searchSpace = ref Unsafe.Add(ref searchSpace, MatchStartOffsetN3); + // All the input bitmaps are Vector128, duplicated 4 times up to Vector512. + // They are stored as Vector512 to lower the overhead of routines that do load the full Vector512. + // When using the Vector128 routine, we just load the first of those duplicates (._lower._lower). Vector128 n0Low = _n0Low._lower._lower, n0High = _n0High._lower._lower; Vector128 n1Low = _n1Low._lower._lower, n1High = _n1High._lower._lower; Vector128 n2Low = _n2Low._lower._lower, n2High = _n2High._lower._lower; + + // As matching is offset by 2 positions (MatchStartOffsetN3), we must remember the result of the previous loop iteration. + // See the full description of TeddyHelper.ProcessInputN3 for more details about why these exist. + // When doing the first loop iteration, there is no previous iteration, so we have to assume that the input did match (AllBitsSet) + // for those positions. This makes it more likely to hit a false-positive at the very beginning, but TryFindMatch will discard them. Vector128 prev0 = Vector128.AllBitsSet; Vector128 prev1 = Vector128.AllBitsSet; Loop: + // Load the input characters and normalize them to their uppercase variant if we're ignoring casing. + // These characters may not be ASCII, but we know that the starting 3 characters of each value are. Vector128 input = TStartCaseSensitivity.TransformInput(LoadAndPack16AsciiChars(ref searchSpace)); + // Find which buckets contain potential matches for each input position. + // For a bucket to be marked as a potential match, its fingerprint must match for all 3 starting characters (all 6 nibbles). (Vector128 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); if (result != Vector128.Zero) @@ -290,6 +317,7 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) } ContinueLoop: + // We haven't found a match. Update the input position and check if we've reached the end. searchSpace = ref Unsafe.Add(ref searchSpace, CharsPerIterationVector128); if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpaceStart)) @@ -301,6 +329,7 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) // We're switching which characters we will process in the next iteration. // prev0 and prev1 no longer point to the characters just before the current input, so we must reset them. + // Just like with the first iteration, we must assume that these positions did match (AllBitsSet). prev0 = Vector128.AllBitsSet; prev1 = Vector128.AllBitsSet; searchSpace = ref lastSearchSpaceStart; @@ -308,6 +337,7 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) goto Loop; CandidateFound: + // We found potential matches, but they may be false-positives, so we must verify each one. if (TryFindMatch(span, ref searchSpace, result, MatchStartOffsetN3, out int offset)) { return offset; @@ -318,6 +348,8 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) [CompExactlyDependsOn(typeof(Avx2))] private int IndexOfAnyN3Avx2(ReadOnlySpan span) { + // See comments in 'IndexOfAnyN3Vector128' above. + // This method is the same, but operates on 32 input characters at a time. Debug.Assert(span.Length >= CharsPerIterationAvx2 + MatchStartOffsetN3); ref char searchSpace = ref MemoryMarshal.GetReference(span); @@ -370,6 +402,8 @@ private int IndexOfAnyN3Avx2(ReadOnlySpan span) [CompExactlyDependsOn(typeof(Avx512BW))] private int IndexOfAnyN3Avx512(ReadOnlySpan span) { + // See comments in 'IndexOfAnyN3Vector128' above. + // This method is the same, but operates on 64 input characters at a time. Debug.Assert(span.Length >= CharsPerIterationAvx512 + MatchStartOffsetN3); ref char searchSpace = ref MemoryMarshal.GetReference(span); @@ -422,20 +456,25 @@ private int IndexOfAnyN3Avx512(ReadOnlySpan span) [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector128 result, int matchStartOffset, out int offsetFromStart) { + // 'resultMask' encodes the input positions where at least one bucket may contain a match. + // These positions are offset by 'matchStartOffset' places. uint resultMask = (~Vector128.Equals(result, Vector128.Zero)).ExtractMostSignificantBits(); do { int matchOffset = BitOperations.TrailingZeroCount(resultMask); + // Calculate where in the input span this potential match begins. ref char matchRef = ref Unsafe.Add(ref searchSpace, matchOffset - matchStartOffset); offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); int lengthRemaining = span.Length - offsetFromStart; + // 'candidateMask' encodes which buckets contain potential matches, starting at 'matchRef'. uint candidateMask = result.GetElementUnsafe(matchOffset); do { + // Verify each bucket to see if we've found a match. int candidateOffset = BitOperations.TrailingZeroCount(candidateMask); object? bucket = _buckets[candidateOffset]; @@ -463,6 +502,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector1 [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector256 result, int matchStartOffset, out int offsetFromStart) { + // See comments in 'TryFindMatch' for Vector128 above. + // This method is the same, but checks the potential matches for 32 input positions. uint resultMask = (~Vector256.Equals(result, Vector256.Zero)).ExtractMostSignificantBits(); do @@ -504,6 +545,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector2 [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector512 result, int matchStartOffset, out int offsetFromStart) { + // See comments in 'TryFindMatch' for Vector128 above. + // This method is the same, but checks the potential matches for 64 input positions. ulong resultMask = (~Vector512.Equals(result, Vector512.Zero)).ExtractMostSignificantBits(); do diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs index 5584d97edc1b9..40a1a8a59902a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs @@ -11,6 +11,7 @@ namespace System.Buffers { internal static class TeddyBucketizer { + // This method is the same as GenerateBucketizedFingerprint below, but each bucket only contains 1 value. public static (Vector512 Low, Vector512 High) GenerateNonBucketizedFingerprint(ReadOnlySpan values, int offset) { Debug.Assert(values.Length <= 8); @@ -37,6 +38,13 @@ public static (Vector512 Low, Vector512 High) GenerateNonBucketizedF return (DuplicateTo512(low), DuplicateTo512(high)); } + // We can have up to 8 buckets, and their positions are encoded by 1 bit each. + // Every bitmap encodes a mapping of each of the possible 16 nibble values into an 8-bit bitmap. + // For example if bucket 0 contains strings ["foo", "bar"], the bitmaps will have the first bit (0th bucket) set like the following: + // 'f' is 0x66, 'b' is 0x62, so n0Low has the bit set at index 2 and 6, n0High has it set at index 6. + // 'o' is 0x6F, 'a' is 0x61, so n1Low has the bit set at index 1 and 15, n1High has it set at index 6. + // 'o' is 0x6F, 'r' is 0x72, so n2Low has the bit set at index 2 and 15, n2High has it set at index 6 and 7. + // We repeat this for each bucket and then OR together the bitmaps (fingerprints) of each bucket to generate a single bitmap for each nibble. public static (Vector512 Low, Vector512 High) GenerateBucketizedFingerprint(string[][] valueBuckets, int offset) { Debug.Assert(valueBuckets.Length <= 8); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs index b7e84dac0980e..291eb627ce2ca 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyHelper.cs @@ -10,11 +10,8 @@ namespace System.Buffers { /// /// Contains the implementation of core vectorized Teddy matching operations. + /// They determine which buckets contain potential matches for each input position. /// - /// - /// TODO: Reworded explanation of how the algorithm works. - /// https://github.com/jneem/teddy#teddy-1 is a good starting point. - /// internal static class TeddyHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -26,15 +23,23 @@ public static (Vector128 Result, Vector128 Prev0) ProcessInputN2( Vector128 n0Low, Vector128 n0High, Vector128 n1Low, Vector128 n1High) { + // See the full description of ProcessInputN3 below for more details. + // This method follows the same pattern as ProcessInputN3, but compares 2 bytes of each bucket at a time instead of 3. + // We are dealing with 4 input nibble bitmaps instead of 6, and only 1 result from the previous iteration instead of 2. (Vector128 low, Vector128 high) = GetNibbles(input); + // Shuffle each nibble with the 2 corresponding bitmaps to determine which positions match any bucket. Vector128 match0 = Shuffle(n0Low, n0High, low, high); Vector128 result1 = Shuffle(n1Low, n1High, low, high); + // RightShift1 shifts the match0 vector to the right by 1 place and shifts in 1 byte from the previous iteration. Vector128 result0 = RightShift1(prev0, match0); + // AND the results together to obtain a list of only buckets that match at all 4 nibble positions. Vector128 result = result0 & result1; + // Return the result and the current matches for byte 0. + // The next loop iteration, 'match0' will be passed back to this method as 'prev0'. return (result, match0); } @@ -46,6 +51,8 @@ public static (Vector256 Result, Vector256 Prev0) ProcessInputN2( Vector256 n0Low, Vector256 n0High, Vector256 n1Low, Vector256 n1High) { + // See comments in 'ProcessInputN2' for Vector128 above. + // This method is the same, but operates on 32 input characters at a time. (Vector256 low, Vector256 high) = GetNibbles(input); Vector256 match0 = Shuffle(n0Low, n0High, low, high); @@ -66,6 +73,8 @@ public static (Vector512 Result, Vector512 Prev0) ProcessInputN2( Vector512 n0Low, Vector512 n0High, Vector512 n1Low, Vector512 n1High) { + // See comments in 'ProcessInputN2' for Vector128 above. + // This method is the same, but operates on 64 input characters at a time. (Vector512 low, Vector512 high) = GetNibbles(input); Vector512 match0 = Shuffle(n0Low, n0High, low, high); @@ -88,17 +97,63 @@ public static (Vector128 Result, Vector128 Prev0, Vector128 Pr Vector128 n1Low, Vector128 n1High, Vector128 n2Low, Vector128 n2High) { + // This is the core operation of the Teddy algorithm that determines which of the buckets contain potential matches. + // Every input bitmap argument (n0Low, n0High, ...) encodes a mapping of each of the possible 16 nibble values into an 8-bit bitmap. + // We test each nibble in the input against these bitmaps to determine which buckets match a given nibble. + // We then AND together these results to obtain only a list of buckets that match at all 6 nibble positions. + // Each byte of the result represents an 8-bit bitmask of buckets that may match at each position. (Vector128 low, Vector128 high) = GetNibbles(input); + // Shuffle each nibble with the 3 corresponding bitmaps to determine which positions match any bucket. Vector128 match0 = Shuffle(n0Low, n0High, low, high); Vector128 match1 = Shuffle(n1Low, n1High, low, high); Vector128 result2 = Shuffle(n2Low, n2High, low, high); + // match0 contain the information for bucket matches at position 0. + // match1 contain the information for bucket matches at position 1. + // result2 contain the information for bucket matches at position 2. + // If we imagine that we only have 1 bucket with 1 string "ABC", the bitmaps we've just obtained encode the following information: + // match0 tells us at which positions we matched the letter 'A' + // match1 tells us at which positions we matched the letter 'B' + // result2 tells us at which positions we matched the letter 'C' + // If input represents the text "BC text ABC text", they would contain: + // input: [B, C, , t, e, x, t, , A, B, C, , t, e, x, t] + // match0: [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] + // match1: [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] + // result2: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // ^ ^ ^ + // Note how the input contains the string ABC, but the matches are not aligned, so we can't just AND them together. + // To solve this, we shift 'match0' to the right by 2 places and 'match1' to the right by 1 place. + // result0: [?, ?, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] + // result1: [?, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] + // result2: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // ^ ^ ^ + // The results are now aligned, but we don't know whether the first two positions matched result0 and result1. + // To replace the missing bytes, we remember the matches from the previous loop iteration, and look at their last 2 bytes. + // If the previous loop iteration ended on the character 'A', we might even have an earlier match. + // For example, if the previous input was "Random strings A": + // prev0: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] + // result0: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // ^ ^ + // We will merge the last two bytes of 'prev0' into 'result0' and the last byte of 'prev1' into 'result1' + // result0: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // result1: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // result2: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // + // RightShift1 and RightShift2 perform the above operation of shifting the match vectors + // to the right by 1 and 2 places and shifting in the bytes from the previous iteration. Vector128 result0 = RightShift2(prev0, match0); Vector128 result1 = RightShift1(prev1, match1); + // AND the results together to obtain a list of only buckets that match at all 6 nibble positions. + // result: [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + // ^ ^ + // Note that we found the match at index 1, even though that match started 2 bytes earlier, at the end of the previous iteration. + // The caller must account for that when verifying potential matches, see 'MatchStartOffsetN3 = 2' in 'AsciiStringSearchValuesTeddyBase'. Vector128 result = result0 & result1 & result2; + // Return the result and the current matches for byte 0 and 1. + // The next loop iteration, 'match0' and 'match1' will be passed back to this method as 'prev0' and 'prev1'. return (result, match0, match1); } @@ -111,6 +166,8 @@ public static (Vector256 Result, Vector256 Prev0, Vector256 Pr Vector256 n1Low, Vector256 n1High, Vector256 n2Low, Vector256 n2High) { + // See comments in 'ProcessInputN3' for Vector128 above. + // This method is the same, but operates on 32 input characters at a time. (Vector256 low, Vector256 high) = GetNibbles(input); Vector256 match0 = Shuffle(n0Low, n0High, low, high); @@ -134,6 +191,8 @@ public static (Vector512 Result, Vector512 Prev0, Vector512 Pr Vector512 n1Low, Vector512 n1High, Vector512 n2Low, Vector512 n2High) { + // See comments in 'ProcessInputN3' for Vector128 above. + // This method is the same, but operates on 64 input characters at a time. (Vector512 low, Vector512 high) = GetNibbles(input); Vector512 match0 = Shuffle(n0Low, n0High, low, high); @@ -148,6 +207,8 @@ public static (Vector512 Result, Vector512 Prev0, Vector512 Pr return (result, match0, match1); } + // Read two Vector512 and concatenate their lower bytes together into a single Vector512. + // On X86, characters above 32767 are turned into 0, but we account for that by not using Teddy if any of the string values contain a 0. [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] [CompExactlyDependsOn(typeof(AdvSimd))] @@ -161,6 +222,8 @@ public static Vector128 LoadAndPack16AsciiChars(ref char source) : AdvSimd.ExtractNarrowingSaturateUpper(AdvSimd.ExtractNarrowingSaturateLower(source0), source1); } + // Read two Vector512 and concatenate their lower bytes together into a single Vector512. + // Characters above 32767 are turned into 0, but we account for that by not using Teddy if any of the string values contain a 0. [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] public static Vector256 LoadAndPack32AsciiChars(ref char source) @@ -173,6 +236,8 @@ public static Vector256 LoadAndPack32AsciiChars(ref char source) return PackedSpanHelpers.FixUpPackedVector256Result(packed); } + // Read two Vector512 and concatenate their lower bytes together into a single Vector512. + // Characters above 32767 are turned into 0, but we account for that by not using Teddy if any of the string values contain a 0. [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx512BW))] public static Vector512 LoadAndPack64AsciiChars(ref char source) @@ -264,6 +329,10 @@ private static Vector128 RightShift1(Vector128 left, Vector128 } else { + // We create a temporary 'leftShifted' vector where the 1st element is the 16th element of the input. + // We then use TBX to shuffle all the elements one place to the left. + // 0xFF is used for the first element to replace it with the one from 'leftShifted'. + Vector128 leftShifted = Vector128.Shuffle(left, Vector128.Create(15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).AsByte()); return AdvSimd.Arm64.VectorTableLookupExtension(leftShifted, right, Vector128.Create(0xFF, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)); } @@ -286,6 +355,10 @@ private static Vector128 RightShift2(Vector128 left, Vector128 } else { + // We create a temporary 'leftShifted' vector where the 1st and 2nd element are the 15th and 16th element of the input. + // We then use TBX to shuffle all the elements two places to the left. + // 0xFF is used for the first two elements to replace them with the ones from 'leftShifted'. + Vector128 leftShifted = Vector128.Shuffle(left, Vector128.Create(14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0).AsByte()); return AdvSimd.Arm64.VectorTableLookupExtension(leftShifted, right, Vector128.Create(0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13)); } From 73ca19e1fe6d32ac8b37eed1a07481aa877a9fcc Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 02:17:40 +0200 Subject: [PATCH 24/43] Improve comments around StringSearchValuesHelper --- .../Strings/Helpers/StringSearchValuesHelper.cs | 9 +++++++++ .../src/System/Text/Ascii.Equality.cs | 4 ---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index f28492dc452f2..4dccccf16dba5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -8,6 +8,8 @@ namespace System.Buffers { + // Provides implementations for helpers shared across multiple SearchValues implementations, + // such as normalizing and matching values under different case sensitivity rules. internal static class StringSearchValuesHelper { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -61,6 +63,7 @@ public interface ICaseSensitivity static abstract bool Equals(ref char matchStart, string candidate); } + // Performs no case transformations. public readonly struct CaseSensitive : ICaseSensitivity { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -80,6 +83,8 @@ public static bool Equals(ref char matchStart, string candidate) => ScalarEquals(ref matchStart, candidate); } + // Transforms inputs to their uppercase variants with the assumption that all input characters are ASCII letters. + // These helpers may produce wrong results for other characters, and the callers must account for that. public readonly struct CaseInsensitiveAsciiLetters : ICaseSensitivity { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -99,6 +104,8 @@ public static bool Equals(ref char matchStart, string candidate) => ScalarEquals(ref matchStart, candidate); } + // Transforms inputs to their uppercase variants with the assumption that all input characters are ASCII. + // These helpers may produce wrong results for non-ASCII inputs, and the callers must account for that. public readonly struct CaseInsensitiveAscii : ICaseSensitivity { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -142,6 +149,8 @@ public static bool Equals(ref char matchStart, string candidate) => ScalarEquals(ref matchStart, candidate); } + // We can't efficiently map non-ASCII inputs to their Ordinal uppercase variants, + // so this helper is only used for the verification of the whole input. public readonly struct CaseInsensitiveUnicode : ICaseSensitivity { public static char TransformInput(char input) => throw new UnreachableException(); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs index 1aa19c95adc94..31df908d9afb0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Equality.cs @@ -189,10 +189,6 @@ public static bool EqualsIgnoreCase(ReadOnlySpan left, ReadOnlySpan => left.Length == right.Length && EqualsIgnoreCase>(ref Unsafe.As(ref MemoryMarshal.GetReference(left)), ref Unsafe.As(ref MemoryMarshal.GetReference(right)), (uint)right.Length); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool EqualsIgnoreCase(ref char left, ref char right, nuint length) => - EqualsIgnoreCase>(ref Unsafe.As(ref left), ref Unsafe.As(ref right), length); - private static bool EqualsIgnoreCase(ref TLeft left, ref TRight right, nuint length) where TLeft : unmanaged, INumberBase where TRight : unmanaged, INumberBase From e672f5f500557400af59b9d8294b7d5f11b62db9 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 02:34:49 +0200 Subject: [PATCH 25/43] Tweak RabinKarp comments --- .../SearchValues/Strings/Helpers/RabinKarp.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index 274285e130965..f7d561c5d51f5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -5,6 +5,7 @@ using System.Globalization; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using static System.Buffers.StringSearchValuesHelper; namespace System.Buffers { @@ -31,7 +32,7 @@ internal readonly struct RabinKarp // We're using nuint as the rolling hash, so we can spread the hash over more bits on 64bit. private static int HashShiftPerElement => IntPtr.Size == 8 ? 2 : 1; - private readonly string[][] _buckets; + private readonly string[]?[] _buckets; private readonly int _hashLength; private readonly nuint _hashUpdateMultiplier; @@ -58,7 +59,7 @@ public RabinKarp(ReadOnlySpan values) return; } - string[][] buckets = _buckets = new string[BucketCount][]; + string[]?[] buckets = _buckets = new string[BucketCount][]; foreach (string value in values) { @@ -71,6 +72,8 @@ public RabinKarp(ReadOnlySpan values) nuint bucket = hash % BucketCount; string[] newBucket; + // Start with a bucket containing 1 element and reallocate larger ones if needed. + // As MaxValues is similar to BucketCount, we will have 1 value per bucket on average. if (buckets[bucket] is string[] existingBucket) { newBucket = new string[existingBucket.Length + 1]; @@ -88,18 +91,19 @@ public RabinKarp(ReadOnlySpan values) [MethodImpl(MethodImplOptions.AggressiveInlining)] public readonly int IndexOfAny(ReadOnlySpan span) - where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, ICaseSensitivity { - return typeof(TCaseSensitivity) == typeof(StringSearchValuesHelper.CaseInsensitiveUnicode) + return typeof(TCaseSensitivity) == typeof(CaseInsensitiveUnicode) ? IndexOfAnyCaseInsensitiveUnicode(span) : IndexOfAnyCore(span); } private readonly int IndexOfAnyCore(ReadOnlySpan span) - where TCaseSensitivity : struct, StringSearchValuesHelper.ICaseSensitivity + where TCaseSensitivity : struct, ICaseSensitivity { - Debug.Assert(typeof(TCaseSensitivity) != typeof(StringSearchValuesHelper.CaseInsensitiveUnicode)); + Debug.Assert(typeof(TCaseSensitivity) != typeof(CaseInsensitiveUnicode)); Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled short inputs."); + Debug.Assert(_buckets is not null); ref char current = ref MemoryMarshal.GetReference(span); @@ -115,15 +119,15 @@ private readonly int IndexOfAnyCore(ReadOnlySpan span) hash = (hash << HashShiftPerElement) + TCaseSensitivity.TransformInput(Unsafe.Add(ref current, i)); } - string[][] buckets = _buckets; + ref string[]? bucketsRef = ref MemoryMarshal.GetArrayDataReference(_buckets); while (true) { - if (Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(buckets), hash % BucketCount) is string[] bucket) + if (Unsafe.Add(ref bucketsRef, hash % BucketCount) is string[] bucket) { int startOffset = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref current) / sizeof(char)); - if (StringSearchValuesHelper.StartsWith(ref current, span.Length - startOffset, bucket)) + if (StartsWith(ref current, span.Length - startOffset, bucket)) { return startOffset; } @@ -162,7 +166,7 @@ private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnlySpan span) Debug.Assert(charsWritten == upperCase.Length); // CaseSensitive instead of CaseInsensitiveUnicode as we've already done the case conversion. - return IndexOfAnyCore(upperCase); + return IndexOfAnyCore(upperCase); } } } From bcf6a8eba81938ad732715bc7e0d4254e17f123e Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 04:59:10 +0200 Subject: [PATCH 26/43] Improve comments around Aho-Corasick --- .../src/System/Globalization/Ordinal.cs | 2 +- .../Strings/Helpers/AhoCorasick.cs | 33 ++++++++++++++- .../Strings/Helpers/AhoCorasickBuilder.cs | 41 +++++++++++++++++-- .../Strings/Helpers/AhoCorasickNode.cs | 21 +++++++--- .../Helpers/StringSearchValuesHelper.cs | 2 + .../Strings/StringSearchValues.cs | 1 + 6 files changed, 90 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index 3c5a38ac17034..36854cd07bab7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -78,7 +78,7 @@ internal static int CompareStringIgnoreCaseNonAscii(ref char strA, int lengthA, return OrdinalCasing.CompareStringIgnoreCase(ref strA, lengthA, ref strB, lengthB); } - internal static bool EqualsIgnoreCase_Vector128(ref char charA, ref char charB, int length) + private static bool EqualsIgnoreCase_Vector128(ref char charA, ref char charB, int length) { Debug.Assert(length >= Vector128.Count); Debug.Assert(Vector128.IsHardwareAccelerated); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 9a747c73ba011..9bcf088a8dc45 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -85,6 +85,9 @@ private readonly int IndexOfAnyCore(ReadOnly int i = 0; FastScan: + Debug.Assert(nodeIndex == 0); + // We are currently in the root node and trying to find the next position of any starting character. + // If all the values start with an ASCII character, use a vectorized helper to quickly skip over characters that can't start a match. if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && typeof(TFastScanVariant) == typeof(IndexOfAnyAsciiFastScan)) { int remainingLength = span.Length - i; @@ -116,6 +119,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), } LoopWithoutRangeCheck: + // Read the next input character and either find the next potential match prefix or transition back to the root node. Debug.Assert(i < span.Length); char c = TCaseSensitivity.TransformInput(Unsafe.Add(ref MemoryMarshal.GetReference(span), i)); @@ -126,11 +130,14 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), if (node.TryGetChild(c, out int childIndex)) { + // We were able to extend the current match. If this node contains a potential match, remember that. nodeIndex = childIndex; int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; if (matchLength != 0) { + // Any result we find from here on out may only be lower (longer match with a start closer to the beginning of the input). + Debug.Assert(result == -1 || result >= i + 1 - matchLength); result = i + 1 - matchLength; } @@ -140,23 +147,30 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), if (nodeIndex == 0) { + // We are back at the root node and none of the values start with the current character. if (result >= 0) { + // If we've already found a match, we can't find an earlier one anymore. This is the result goto Return; } + // Go back to searching for the next possible starting character. i++; goto FastScan; } + // Follow the next suffix link. nodeIndex = node.SuffixLink; if (nodeIndex < 0) { + // A node with a suffix link of -1 indicates a match, see AhoCorasickBuilder.AddSuffixLinks. Debug.Assert(nodeIndex == -1); Debug.Assert(result >= 0); goto Return; } + + // Try to match the current character again at the suffix link node. } Return: @@ -176,10 +190,14 @@ private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnly char lowSurrogateUpper = LowSurrogateNotSet; FastScan: + // We are currently in the root node and trying to find the next position of any starting character. + // If all the values start with an ASCII character, use a vectorized helper to quickly skip over characters that can't start a match. if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && typeof(TFastScanVariant) == typeof(IndexOfAnyAsciiFastScan)) { if (lowSurrogateUpper != LowSurrogateNotSet) { + // We read a surrogate pair in the previous loop iteration and processed the high surrogate. + // Continue with the stored low surrogate. goto LoopWithoutRangeCheck; } @@ -209,15 +227,18 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), } LoopWithoutRangeCheck: + // Read the next input character and either find the next potential match prefix or transition back to the root node. Debug.Assert(i < span.Length); char c; if (lowSurrogateUpper != LowSurrogateNotSet) { + // We have just processed the high surrogate. Continue with the low surrogate we read in the previous iteration. c = lowSurrogateUpper; lowSurrogateUpper = LowSurrogateNotSet; } else { + // Read the next character, check if it's a high surrogate, and transform it to its Ordinal uppercase representation. c = Unsafe.Add(ref MemoryMarshal.GetReference(span), i); char lowSurrogate; @@ -242,7 +263,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), } #if DEBUG - // This logic must match Ordinal.ToUpperOrdinal exactly. + // The above logic must match Ordinal.ToUpperOrdinal exactly. Span destination = new char[2]; // Avoid stackalloc in a loop Ordinal.ToUpperOrdinal(span.Slice(i, i + 1 == span.Length ? 1 : 2), destination); Debug.Assert(c == destination[0]); @@ -257,11 +278,14 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), if (node.TryGetChild(c, out int childIndex)) { + // We were able to extend the current match. If this node contains a potential match, remember that. nodeIndex = childIndex; int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; if (matchLength != 0) { + // Any result we find from here on out may only be lower (longer match with a start closer to the beginning of the input). + Debug.Assert(result == -1 || result >= i + 1 - matchLength); result = i + 1 - matchLength; } @@ -271,23 +295,30 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), if (nodeIndex == 0) { + // We are back at the root node and none of the values start with the current character. if (result >= 0) { + // If we've already found a match, we can't find an earlier one anymore. This is the result goto Return; } + // Go back to searching for the next possible starting character. i++; goto FastScan; } + // Follow the next suffix link. nodeIndex = node.SuffixLink; if (nodeIndex < 0) { + // A node with a suffix link of -1 indicates a match, see AhoCorasickBuilder.AddSuffixLinks. Debug.Assert(nodeIndex == -1); Debug.Assert(result >= 0); goto Return; } + + // Try to match the current character again at the suffix link node. } Return: diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs index 7449c40855699..686525e19e466 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickBuilder.cs @@ -90,7 +90,7 @@ private void BuildTrie(ref HashSet? unreachableValues) if (node.MatchLength != 0) { // A previous value is an exact prefix of this one. - // We're looking for the index of the first match, not necessarily the longest one, we can skip this value. + // We're looking for the index of the first match, not necessarily the longest one, so we can skip this value. // We've already normalized the values, so we can do ordinal comparisons here. unreachableValues ??= new HashSet(StringComparer.Ordinal); unreachableValues.Add(value); @@ -108,6 +108,33 @@ private void BuildTrie(ref HashSet? unreachableValues) private void AddSuffixLinks() { + // Besides the list of children which continue the current value, each node also contains a suffix link + // which points to the node with the longest suffix of the current node. + // When we're searching and can't find a child to extend the current string with, we will follow + // suffix links to find the longest string that does match up until the current point. + // + // For example if we have strings "DOTNET" and "OTTER", we want + // the 'O' and 'T' in "dotnet" to point into 'O' and 'T' in "OTTER". + // If our text contains the word "dotter", we will walk it character by character. + // Once we get to "DOTNET" and read the next character 'T', we can no longer continue with "DOTNET", + // and will instead follow the suffix link to "ot" in "OTTER" where we can continue the search. + // + // We also remember when a node's suffix link points to the end of a different value, such that it is itself a match. + // If we also had the word "POTTERY", the 'R' would contain a suffix link to the 'R' in "OTTER", + // but also mark that it is already a length=5 match. + // + // +---> D O T N E T + // | | | + // | +--+ | + // root--+ | | + // | | +--+ + // | v v + // +---> O T T E R + // | ^ ^ ^ ^ ^ + // | | | | | | -- this is also a length=5 match + // | | | | | | + // +> P O T T E R Y + var queue = new Queue<(char Char, int Index)>(); queue.Enqueue(((char)0, 0)); @@ -117,6 +144,8 @@ private void AddSuffixLinks() int parent = _parents[trieNode.Index]; int suffixLink = _nodes[parent].SuffixLink; + // If this node doesn't represent the first character of a value (doesn't immediately follow the root node), + // it may have a have a non-zero suffix link. if (parent != 0) { while (suffixLink >= 0) @@ -140,10 +169,13 @@ private void AddSuffixLinks() if (node.MatchLength != 0) { + // This node represents the end of a match. + // Mark it in a special way we can recognize when searching. node.SuffixLink = -1; - // If a node is a match, there's no need to assign suffix links to its children. - // If a child does not match, such that we would look at its suffix link, we already saw an earlier match node. + // If a node is a match, there is no need to assign suffix links to its children. + // If a child does not match, such that we would look at its suffix link, + // we have already saw an earlier match node that is definitely the earliest possible match. } else { @@ -151,6 +183,7 @@ private void AddSuffixLinks() if (suffixLink >= 0) { + // Remember if this node's suffix link points to a node that is itself a match. node.MatchLength = _nodes[suffixLink].MatchLength; } @@ -159,6 +192,8 @@ private void AddSuffixLinks() } } + // If all the values start with ASCII characters, we can use IndexOfAnyAsciiSearcher + // to quickly skip to the next possible starting location in the input. private void GenerateStartingAsciiCharsBitmap() { scoped ValueListBuilder startingChars = new ValueListBuilder(stackalloc char[128]); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs index 512b30fcf37ec..d5d0c03c67c86 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasickNode.cs @@ -15,6 +15,8 @@ internal struct AhoCorasickNode public int SuffixLink; public int MatchLength; + // This is not a radix tree so we may have a lot of very sparse nodes (single child). + // We save 1 child separately to avoid allocating a separate collection in such cases. private int _firstChildChar; private int _firstChildIndex; private object _children; // Either int[] or Dictionary @@ -89,6 +91,10 @@ public readonly void AddChildrenToQueue(Queue<(char Char, int Index)> queue) queue.Enqueue((childChar, childIndex)); } } + else + { + Debug.Assert(ReferenceEquals(_children, EmptyChildrenSentinel)); + } } } @@ -100,6 +106,8 @@ public void OptimizeChildren() float frequency = -2; + // We have the _firstChildChar field that will always be checked first. + // Improve throughput by setting it to the child character with the highest frequency. foreach ((char childChar, int childIndex) in children) { float newFrequency = char.IsAscii(childChar) ? CharacterFrequencyHelper.AsciiFrequency[childChar] : -1; @@ -122,7 +130,10 @@ public void OptimizeChildren() static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true)] out int[]? table) { - // Sacrifice some memory usage in exchange for faster lookup performance + // We can use either a Dictionary or int[] to map child characters to node indexes. + // int[] is generally faster but consumes more memory for characters with high values. + // We try to find the right balance between memory usage and lookup performance. + // Currently we will sacrifice up to ~2x the memory consumption to use int[] for faster lookups. const int AcceptableSizeMultiplier = 2; Debug.Assert(children.Count > 0); @@ -134,8 +145,8 @@ static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true maxValue = Math.Max(maxValue, childChar); } - int tableSize = TableBytesEstimate(maxValue); - int dictionarySize = DictionaryBytesEstimate(children.Count); + int tableSize = TableMemoryFootprintBytesEstimate(maxValue); + int dictionarySize = DictionaryMemoryFootprintBytesEstimate(children.Count); if (tableSize > dictionarySize * AcceptableSizeMultiplier) { @@ -154,7 +165,7 @@ static bool TryCreateJumpTable(Dictionary children, [NotNullWhen(true return true; - static int TableBytesEstimate(int maxValue) + static int TableMemoryFootprintBytesEstimate(int maxValue) { // An approximate number of bytes consumed by an // int[] table with a known number of entries. @@ -162,7 +173,7 @@ static int TableBytesEstimate(int maxValue) return 32 + (maxValue * sizeof(int)); } - static int DictionaryBytesEstimate(int childCount) + static int DictionaryMemoryFootprintBytesEstimate(int childCount) { // An approximate number of bytes consumed by a // Dictionary with a known number of entries. diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index 4dccccf16dba5..436b265253ef5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -31,6 +31,8 @@ public static bool StartsWith(ref char matchStart, int lengthR public static bool StartsWith(ref char matchStart, int lengthRemaining, string candidate) where TCaseSensitivity : struct, ICaseSensitivity { + Debug.Assert(lengthRemaining > 0); + if (lengthRemaining < candidate.Length) { return false; diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index c26f0b26333aa..98bbf0c40bb31 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -123,6 +123,7 @@ private static SearchValues CreateFromNormalizedValues( return searchValues; } + // Fall back to Aho-Corasick for all other multi-value sets. AhoCorasick ahoCorasick = ahoCorasickBuilder.Build(); if (!ignoreCase) From de3dedd3aeff711dba213a3c46c945b9b706306a Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 05:06:22 +0200 Subject: [PATCH 27/43] Remove some SearchValues indirection --- .../src/System/MemoryExtensions.cs | 44 +++++++++++++++---- .../src/System/SearchValues/SearchValues.T.cs | 44 ------------------- 2 files changed, 36 insertions(+), 52 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index df69789583143..4d8ff44fd202e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -1035,8 +1035,15 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), /// If all of the values are in , returns -1. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int IndexOfAnyExcept(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => - SearchValues.IndexOfAnyExcept(span, values); + public static int IndexOfAnyExcept(this ReadOnlySpan span, SearchValues values) where T : IEquatable? + { + if (values is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); + } + + return values.IndexOfAnyExcept(span); + } /// Searches for the last index of any value other than the specified . /// The type of the span and values. @@ -1338,8 +1345,15 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), /// If all of the values are in , returns -1. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LastIndexOfAnyExcept(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => - SearchValues.LastIndexOfAnyExcept(span, values); + public static int LastIndexOfAnyExcept(this ReadOnlySpan span, SearchValues values) where T : IEquatable? + { + if (values is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); + } + + return values.LastIndexOfAnyExcept(span); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -2081,8 +2095,15 @@ public static unsafe int IndexOfAny(this ReadOnlySpan span, ReadOnlySpanThe span to search. /// The set of values to search for. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int IndexOfAny(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => - SearchValues.IndexOfAny(span, values); + public static int IndexOfAny(this ReadOnlySpan span, SearchValues values) where T : IEquatable? + { + if (values is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); + } + + return values.IndexOfAny(span); + } /// /// Searches for the first index of any of the specified substring values. @@ -2371,8 +2392,15 @@ public static unsafe int LastIndexOfAny(this ReadOnlySpan span, ReadOnlySp /// The span to search. /// The set of values to search for. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LastIndexOfAny(this ReadOnlySpan span, SearchValues values) where T : IEquatable? => - SearchValues.LastIndexOfAny(span, values); + public static int LastIndexOfAny(this ReadOnlySpan span, SearchValues values) where T : IEquatable? + { + if (values is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); + } + + return values.LastIndexOfAny(span); + } /// /// Determines whether two sequences are equal by comparing the elements using IEquatable{T}.Equals(T). diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs index a1e68fcb044c3..f77409e9799c8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.T.cs @@ -42,50 +42,6 @@ private protected SearchValues() { } // This is only implemented and used by SearchValues. internal virtual int IndexOfAnyMultiString(ReadOnlySpan span) => throw new UnreachableException(); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfAny(ReadOnlySpan span, SearchValues values) - { - if (values is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); - } - - return values.IndexOfAny(span); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int IndexOfAnyExcept(ReadOnlySpan span, SearchValues values) - { - if (values is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); - } - - return values.IndexOfAnyExcept(span); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int LastIndexOfAny(ReadOnlySpan span, SearchValues values) - { - if (values is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); - } - - return values.LastIndexOfAny(span); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int LastIndexOfAnyExcept(ReadOnlySpan span, SearchValues values) - { - if (values is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.values); - } - - return values.LastIndexOfAnyExcept(span); - } - private string DebuggerDisplay { get From 9d02edbb03efc9b79a0a77bc4792686da37eaeaa Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 05:14:46 +0200 Subject: [PATCH 28/43] Remove AggressiveInlining in a few places --- .../SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs | 2 -- .../Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs | 1 - .../Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs | 1 - .../Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs | 1 - .../Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs | 1 - 5 files changed, 6 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index f5c3c37f35b66..4ae7696144e81 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -74,7 +74,6 @@ protected AsciiStringSearchValuesTeddyBase(string[][] buckets, ReadOnlySpan span) @@ -95,7 +94,6 @@ protected int IndexOfAnyN2(ReadOnlySpan span) return IndexOfAnyN2Vector128(span); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] protected int IndexOfAnyN3(ReadOnlySpan span) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs index be7a7140134b2..c8bb2ed30c0ef 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN2.cs @@ -16,7 +16,6 @@ public AsciiStringSearchValuesTeddyBucketizedN2(string[][] buckets, ReadOnlySpan : base(buckets, values, uniqueValues, n: 2) { } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] internal override int IndexOfAnyMultiString(ReadOnlySpan span) => diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs index 2698d02245aa0..96c001f5a758b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBucketizedN3.cs @@ -16,7 +16,6 @@ public AsciiStringSearchValuesTeddyBucketizedN3(string[][] buckets, ReadOnlySpan : base(buckets, values, uniqueValues, n: 3) { } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] internal override int IndexOfAnyMultiString(ReadOnlySpan span) => diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs index 1dbfa78efd88d..af077ad77eba1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN2.cs @@ -16,7 +16,6 @@ public AsciiStringSearchValuesTeddyNonBucketizedN2(ReadOnlySpan values, : base(values, uniqueValues, n: 2) { } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] internal override int IndexOfAnyMultiString(ReadOnlySpan span) => diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs index 63efaa814514c..5e0139b23ba33 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyNonBucketizedN3.cs @@ -16,7 +16,6 @@ public AsciiStringSearchValuesTeddyNonBucketizedN3(ReadOnlySpan values, : base(values, uniqueValues, n: 3) { } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] internal override int IndexOfAnyMultiString(ReadOnlySpan span) => From 34112898eca459a1e10c630895de8e51c98411f6 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 06:20:36 +0200 Subject: [PATCH 29/43] Improve comments around ThreeChars SearchValues --- .../SingleStringSearchValuesThreeChars.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index 98d4b0c481bf1..8b9f0f3015311 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -13,6 +13,7 @@ namespace System.Buffers { // Based on SpanHelpers.IndexOf(ref char, int, ref char, int) // This implementation uses 3 precomputed anchor points when searching. + // This implementation may also be used for length=2 values, in which case two anchors point at the same position. internal sealed class SingleStringSearchValuesThreeChars : StringSearchValuesBase where TCaseSensitivity : struct, ICaseSensitivity { @@ -28,7 +29,9 @@ internal sealed class SingleStringSearchValuesThreeChars : Str public SingleStringSearchValuesThreeChars(string value, HashSet uniqueValues) : base(uniqueValues) { + // We could have more than one entry in 'uniqueValues' if this value is an exact prefix of all the others. Debug.Assert(value.Length > 1); + Debug.Assert(uniqueValues.Count >= 1); bool ignoreCase = typeof(TCaseSensitivity) != typeof(CaseSensitive); @@ -82,6 +85,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector512 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); if (result != Vector512.Zero) @@ -90,6 +94,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) } LoopFooter: + // We haven't found a match. Update the input position and check if we've reached the end. searchSpace = ref Unsafe.Add(ref searchSpace, Vector512.Count); if (Unsafe.IsAddressGreaterThan(ref searchSpace, ref lastSearchSpace)) @@ -99,12 +104,14 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) return -1; } + // We have fewer than 32 characters remaining. Adjust the input position such that we will do one last loop iteration. searchSpace = ref lastSearchSpace; } continue; CandidateFound: + // We found potential matches, but they may be false-positives, so we must verify each one. if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; @@ -122,6 +129,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector256 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); if (result != Vector256.Zero) @@ -139,12 +147,14 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) return -1; } + // We have fewer than 16 characters remaining. Adjust the input position such that we will do one last loop iteration. searchSpace = ref lastSearchSpace; } continue; CandidateFound: + // We found potential matches, but they may be false-positives, so we must verify each one. if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; @@ -162,6 +172,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector128 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); if (result != Vector128.Zero) @@ -179,12 +190,14 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) return -1; } + // We have fewer than 8 characters remaining. Adjust the input position such that we will do one last loop iteration. searchSpace = ref lastSearchSpace; } continue; CandidateFound: + // We found potential matches, but they may be false-positives, so we must verify each one. if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; @@ -194,6 +207,8 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) } ShortInput: + Debug.Assert(searchSpaceLength < Vector128.Count); + string value = _value; char valueHead = value.GetRawStringData(); @@ -201,6 +216,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) { ref char cur = ref Unsafe.Add(ref searchSpace, i); + // CaseInsensitiveUnicode doesn't support single-character transformations, so we skip checking the first character first. if ((typeof(TCaseSensitivity) == typeof(CaseInsensitiveUnicode) || TCaseSensitivity.TransformInput(cur) == valueHead) && TCaseSensitivity.Equals(ref cur, value)) { @@ -214,20 +230,27 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector128 ch1, Vector128 ch2, Vector128 ch3) { + // Load 3 vectors from the input. + // One from the current search space, the other two at an offset based on the distance of those characters from the first one. if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) { Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace)); Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16()); Vector128 cmpCh3 = Vector128.Equals(ch3, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16()); + // AND all 3 together to get a mask of possible match positions that match in at least 3 places. return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); } else { + // For each, AND the value with ~0x20 so that letters are uppercased. + // For characters that aren't ASCII letters, this may produce wrong results, but only false-positives. + // We will take care of those in the verification step if the other characters also indicate a possible match. Vector128 caseConversion = Vector128.Create(CaseConversionMask); Vector128 cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref searchSpace) & caseConversion); Vector128 cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch2ByteOffset).AsUInt16() & caseConversion); Vector128 cmpCh3 = Vector128.Equals(ch3, Vector128.LoadUnsafe(ref Unsafe.As(ref searchSpace), ch3ByteOffset).AsUInt16() & caseConversion); + // AND all 3 together to get a mask of possible match positions that likely match in at least 3 places. return (cmpCh1 & cmpCh2 & cmpCh3).AsByte(); } } @@ -235,6 +258,8 @@ private static Vector128 GetComparisonResult(ref char searchSpace, nuint c [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector256 ch1, Vector256 ch2, Vector256 ch3) { + // See comments in 'GetComparisonResult' for Vector128 above. + // This method is the same, but operates on 32 input characters at a time. if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) { Vector256 cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref searchSpace)); @@ -256,6 +281,8 @@ private static Vector256 GetComparisonResult(ref char searchSpace, nuint c [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector512 GetComparisonResult(ref char searchSpace, nuint ch2ByteOffset, nuint ch3ByteOffset, Vector512 ch1, Vector512 ch2, Vector512 ch3) { + // See comments in 'GetComparisonResult' for Vector128 above. + // This method is the same, but operates on 64 input characters at a time. if (typeof(TCaseSensitivity) == typeof(CaseSensitive)) { Vector512 cmpCh1 = Vector512.Equals(ch1, Vector512.LoadUnsafe(ref searchSpace)); @@ -277,6 +304,8 @@ private static Vector512 GetComparisonResult(ref char searchSpace, nuint c [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask, out int offsetFromStart) { + // 'mask' encodes the input positions where at least 3 characters likely matched. + // Verify each one to see if we've found a match, otherwise return back to the vectorized loop. do { int bitPos = BitOperations.TrailingZeroCount(mask); @@ -301,6 +330,8 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mask, out int offsetFromStart) { + // 'mask' encodes the input positions where at least 3 characters likely matched. + // Verify each one to see if we've found a match, otherwise return back to the vectorized loop. do { int bitPos = BitOperations.TrailingZeroCount(mask); From d4f63d6fed7026ae9770384efdb1960994c069b9 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 23 Jul 2023 08:03:31 +0200 Subject: [PATCH 30/43] Fix assert and code typo --- .../src/System/SearchValues/Strings/Helpers/RabinKarp.cs | 2 +- .../SearchValues/Strings/Helpers/StringSearchValuesHelper.cs | 2 +- .../SearchValues/Strings/SingleStringSearchValuesThreeChars.cs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index f7d561c5d51f5..7c5de9c1d0b88 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -103,7 +103,6 @@ private readonly int IndexOfAnyCore(ReadOnlySpan span) { Debug.Assert(typeof(TCaseSensitivity) != typeof(CaseInsensitiveUnicode)); Debug.Assert(span.Length <= MaxInputLength, "Teddy should have handled short inputs."); - Debug.Assert(_buckets is not null); ref char current = ref MemoryMarshal.GetReference(span); @@ -119,6 +118,7 @@ private readonly int IndexOfAnyCore(ReadOnlySpan span) hash = (hash << HashShiftPerElement) + TCaseSensitivity.TransformInput(Unsafe.Add(ref current, i)); } + Debug.Assert(_buckets is not null); ref string[]? bucketsRef = ref MemoryMarshal.GetArrayDataReference(_buckets); while (true) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index 436b265253ef5..e4a2af274784b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -47,7 +47,7 @@ private static bool ScalarEquals(ref char matchStart, string c { for (int i = 0; i < candidate.Length; i++) { - if (TCaseSensitivity.TransformInput(candidate[i]) != Unsafe.Add(ref matchStart, i)) + if (TCaseSensitivity.TransformInput(Unsafe.Add(ref matchStart, i)) != candidate[i]) { return false; } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index 8b9f0f3015311..827287ceaf909 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -207,8 +207,6 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) } ShortInput: - Debug.Assert(searchSpaceLength < Vector128.Count); - string value = _value; char valueHead = value.GetRawStringData(); From 5cb16fa37f24433c452f387ab66794998798920d Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 24 Jul 2023 20:11:31 +0200 Subject: [PATCH 31/43] Reduce overhead for creating single-value SearchValues --- .../SingleStringSearchValuesThreeChars.cs | 61 ++++++++++++++++--- .../Strings/StringSearchValues.cs | 30 +++++---- 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index 827287ceaf909..d80090475c94b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; @@ -14,7 +13,7 @@ namespace System.Buffers // Based on SpanHelpers.IndexOf(ref char, int, ref char, int) // This implementation uses 3 precomputed anchor points when searching. // This implementation may also be used for length=2 values, in which case two anchors point at the same position. - internal sealed class SingleStringSearchValuesThreeChars : StringSearchValuesBase + internal sealed class SingleStringSearchValuesThreeChars : SearchValues where TCaseSensitivity : struct, ICaseSensitivity { private const ushort CaseConversionMask = unchecked((ushort)~0x20); @@ -27,15 +26,14 @@ internal sealed class SingleStringSearchValuesThreeChars : Str private readonly ushort _ch2; private readonly ushort _ch3; - public SingleStringSearchValuesThreeChars(string value, HashSet uniqueValues) : base(uniqueValues) + private static bool IgnoreCase => typeof(TCaseSensitivity) != typeof(CaseSensitive); + + public SingleStringSearchValuesThreeChars(string value) { // We could have more than one entry in 'uniqueValues' if this value is an exact prefix of all the others. Debug.Assert(value.Length > 1); - Debug.Assert(uniqueValues.Count >= 1); - - bool ignoreCase = typeof(TCaseSensitivity) != typeof(CaseSensitive); - CharacterFrequencyHelper.GetSingleStringMultiCharacterOffsets(value, ignoreCase, out int ch2Offset, out int ch3Offset); + CharacterFrequencyHelper.GetSingleStringMultiCharacterOffsets(value, IgnoreCase, out int ch2Offset, out int ch3Offset); Debug.Assert(ch3Offset == 0 || ch3Offset > ch2Offset); @@ -46,7 +44,7 @@ public SingleStringSearchValuesThreeChars(string value, HashSet uniqueVa _ch2 = value[ch2Offset]; _ch3 = value[ch3Offset]; - if (ignoreCase) + if (IgnoreCase) { _ch1 &= CaseConversionMask; _ch2 &= CaseConversionMask; @@ -350,5 +348,52 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mas offsetFromStart = 0; return false; } + + + internal override bool ContainsCore(string value) => + _value.Equals(value, IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); + + internal override string[] GetValues() => + new string[] { _value }; + + internal override int IndexOfAny(ReadOnlySpan span) => + IndexOfAny(span); + + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + IndexOfAny(span); + + internal override int LastIndexOfAny(ReadOnlySpan span) => + LastIndexOfAny(span); + + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + LastIndexOfAny(span); + + private int IndexOfAny(ReadOnlySpan span) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + for (int i = 0; i < span.Length; i++) + { + if (TNegator.NegateIfNeeded(ContainsCore(span[i]))) + { + return i; + } + } + + return -1; + } + + private int LastIndexOfAny(ReadOnlySpan span) + where TNegator : struct, IndexOfAnyAsciiSearcher.INegator + { + for (int i = span.Length - 1; i >= 0; i--) + { + if (TNegator.NegateIfNeeded(ContainsCore(span[i]))) + { + return i; + } + } + + return -1; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 98bbf0c40bb31..a849f98f44e0f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -24,6 +24,17 @@ public static SearchValues Create(ReadOnlySpan values, bool igno return new EmptySearchValues(); } + if (values.Length == 1) + { + // Avoid additional overheads for single-value inputs. + string value = values[0]; + ArgumentNullException.ThrowIfNull(value, nameof(values)); + string normalizedValue = NormalizeIfNeeded(value, ignoreCase); + + AnalyzeValues(new ReadOnlySpan(normalizedValue), ref ignoreCase, out bool ascii, out bool asciiLettersOnly, out _, out _); + return CreateForSingleValue(normalizedValue, uniqueValues: null, ignoreCase, ascii, asciiLettersOnly); + } + var uniqueValues = new HashSet(values.Length, ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); foreach (string value in values) @@ -46,13 +57,6 @@ public static SearchValues Create(ReadOnlySpan values, bool igno } Debug.Assert(i == normalizedValues.Length); - if (normalizedValues.Length == 1) - { - // Avoid the overhead of building the AhoCorasick trie for single-value inputs. - AnalyzeValues(normalizedValues, ref ignoreCase, out bool ascii, out bool asciiLettersOnly, out _, out _); - return CreateForSingleValue(normalizedValues[0], uniqueValues, ignoreCase, ascii, asciiLettersOnly); - } - // Aho-Corasick's ctor expects values to be sorted by length. normalizedValues.Sort(static (a, b) => a.Length.CompareTo(b.Length)); @@ -285,7 +289,7 @@ private static SearchValues PickTeddyImplementation CreateForSingleValue( string value, - HashSet uniqueValues, + HashSet? uniqueValues, bool ignoreCase, bool allAscii, bool asciiLettersOnly) @@ -297,26 +301,28 @@ private static SearchValues CreateForSingleValue( { if (!ignoreCase) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value); } if (asciiLettersOnly) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value); } if (allAscii) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value); } // When ignoring casing, all anchor chars we search for must be ASCII. if (char.IsAscii(value[0]) && value.AsSpan().LastIndexOfAnyInRange((char)0, (char)127) > 0) { - return new SingleStringSearchValuesThreeChars(value, uniqueValues); + return new SingleStringSearchValuesThreeChars(value); } } + uniqueValues ??= new HashSet(1, ignoreCase ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal) { value }; + return ignoreCase ? new SingleStringSearchValuesFallback(value, uniqueValues) : new SingleStringSearchValuesFallback(value, uniqueValues); From e66d8d9d10964c5d835fbfaa0764b2a415f4b4c2 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 24 Jul 2023 20:16:35 +0200 Subject: [PATCH 32/43] Reword TODO to 'Potential optimization' --- .../System/SearchValues/Strings/Helpers/TeddyBucketizer.cs | 2 +- .../src/System/SearchValues/Strings/StringSearchValues.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs index 40a1a8a59902a..c277e6a8f72af 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/TeddyBucketizer.cs @@ -107,7 +107,7 @@ public static string[][] Bucketize(ReadOnlySpan values, int bucketCount, if (!prefixToBucket.TryGetValue(prefix, out int bucketIndex)) { - // TODO: We currently merge values with different prefixes into buckets randomly (round-robin). + // Potential optimization: We currently merge values with different prefixes into buckets randomly (round-robin). // We could employ a more sophisticated strategy here, e.g. by trying to minimize the number of // values in each bucket, or by minimizing the PopCount of final merged fingerprints. // Example of the latter: https://gist.github.com/MihaZupan/831324d1d646b69ae0ba4b54e3446a49 diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index a849f98f44e0f..95490a43a3ed3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -272,7 +272,7 @@ private static SearchValues PickTeddyImplementation Date: Mon, 24 Jul 2023 20:21:31 +0200 Subject: [PATCH 33/43] Add a few more test cases for single values --- src/libraries/System.Memory/tests/Span/StringSearchValues.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index 15dedc9b8deb5..dd3067535b958 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -20,8 +20,13 @@ public static class StringSearchValuesTests public static bool CanTestNls => RemoteExecutor.IsSupported && OperatingSystem.IsWindows(); [Theory] + [InlineData(StringComparison.Ordinal, "a")] + [InlineData(StringComparison.Ordinal, "A")] [InlineData(StringComparison.Ordinal, "a", "ab", "abc", "bc")] [InlineData(StringComparison.Ordinal, "A", "ab", "aBc", "Bc")] + [InlineData(StringComparison.OrdinalIgnoreCase, "a")] + [InlineData(StringComparison.OrdinalIgnoreCase, "A")] + [InlineData(StringComparison.OrdinalIgnoreCase, "A", "a")] [InlineData(StringComparison.OrdinalIgnoreCase, "a", "Ab", "abc", "bC")] public static void Values_ImplementsSearchValuesBase(StringComparison comparisonType, params string[] values) { From d4eca96692b1fd48ecbcecb0c92bf94646f4c807 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Mon, 24 Jul 2023 21:06:29 +0200 Subject: [PATCH 34/43] Combine a-z and !Ascii checks --- .../src/System/SearchValues/Strings/StringSearchValues.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 95490a43a3ed3..4e07b3de8e827 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -17,6 +17,9 @@ internal static class StringSearchValues private static readonly SearchValues s_asciiLetters = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + private static readonly SearchValues s_allAsciiExceptLowercase = + SearchValues.Create("\0\u0001\u0002\u0003\u0004\u0005\u0006\a\b\t\n\v\f\r\u000E\u000F\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`{|}~\u007F"); + public static SearchValues Create(ReadOnlySpan values, bool ignoreCase) { if (values.Length == 0) @@ -78,7 +81,7 @@ public static SearchValues Create(ReadOnlySpan values, bool igno static string NormalizeIfNeeded(string value, bool ignoreCase) { - if (ignoreCase && (value.AsSpan().ContainsAnyInRange('a', 'z') || !Ascii.IsValid(value))) + if (ignoreCase && value.AsSpan().ContainsAnyExcept(s_allAsciiExceptLowercase)) { string upperCase = string.FastAllocateString(value.Length); int charsWritten = Ordinal.ToUpperOrdinal(value, new Span(ref upperCase.GetRawStringData(), upperCase.Length)); From f4579750ecb1fadf06a202e7bc0382c259eb8c78 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Tue, 25 Jul 2023 16:02:09 +0200 Subject: [PATCH 35/43] Add more asserts around read pointer manipulation --- .../AsciiStringSearchValuesTeddyBase.cs | 12 +++++++++ .../Strings/Helpers/AhoCorasick.cs | 6 +++-- .../SearchValues/Strings/Helpers/RabinKarp.cs | 2 ++ .../Helpers/StringSearchValuesHelper.cs | 23 ++++++++++++++++ .../SingleStringSearchValuesThreeChars.cs | 26 +++++++++++++++---- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index 4ae7696144e81..348a0a0ec2bf8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -135,6 +135,7 @@ private int IndexOfAnyN2Vector128(ReadOnlySpan span) Vector128 prev0 = Vector128.AllBitsSet; Loop: + ValidateReadPosition(span, ref searchSpace); Vector128 input = TStartCaseSensitivity.TransformInput(LoadAndPack16AsciiChars(ref searchSpace)); (Vector128 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); @@ -186,6 +187,7 @@ private int IndexOfAnyN2Avx2(ReadOnlySpan span) Vector256 prev0 = Vector256.AllBitsSet; Loop: + ValidateReadPosition(span, ref searchSpace); Vector256 input = TStartCaseSensitivity.TransformInput(LoadAndPack32AsciiChars(ref searchSpace)); (Vector256 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); @@ -237,6 +239,7 @@ private int IndexOfAnyN2Avx512(ReadOnlySpan span) Vector512 prev0 = Vector512.AllBitsSet; Loop: + ValidateReadPosition(span, ref searchSpace); Vector512 input = TStartCaseSensitivity.TransformInput(LoadAndPack64AsciiChars(ref searchSpace)); (Vector512 result, prev0) = ProcessInputN2(input, prev0, n0Low, n0High, n1Low, n1High); @@ -303,6 +306,7 @@ private int IndexOfAnyN3Vector128(ReadOnlySpan span) Loop: // Load the input characters and normalize them to their uppercase variant if we're ignoring casing. // These characters may not be ASCII, but we know that the starting 3 characters of each value are. + ValidateReadPosition(span, ref searchSpace); Vector128 input = TStartCaseSensitivity.TransformInput(LoadAndPack16AsciiChars(ref searchSpace)); // Find which buckets contain potential matches for each input position. @@ -362,6 +366,7 @@ private int IndexOfAnyN3Avx2(ReadOnlySpan span) Vector256 prev1 = Vector256.AllBitsSet; Loop: + ValidateReadPosition(span, ref searchSpace); Vector256 input = TStartCaseSensitivity.TransformInput(LoadAndPack32AsciiChars(ref searchSpace)); (Vector256 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); @@ -416,6 +421,7 @@ private int IndexOfAnyN3Avx512(ReadOnlySpan span) Vector512 prev1 = Vector512.AllBitsSet; Loop: + ValidateReadPosition(span, ref searchSpace); Vector512 input = TStartCaseSensitivity.TransformInput(LoadAndPack64AsciiChars(ref searchSpace)); (Vector512 result, prev0, prev1) = ProcessInputN3(input, prev0, prev1, n0Low, n0High, n1Low, n1High, n2Low, n2High); @@ -467,6 +473,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector1 offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); int lengthRemaining = span.Length - offsetFromStart; + ValidateReadPosition(span, ref matchRef, lengthRemaining); + // 'candidateMask' encodes which buckets contain potential matches, starting at 'matchRef'. uint candidateMask = result.GetElementUnsafe(matchOffset); @@ -512,6 +520,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector2 offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); int lengthRemaining = span.Length - offsetFromStart; + ValidateReadPosition(span, ref matchRef, lengthRemaining); + uint candidateMask = result.GetElementUnsafe(matchOffset); do @@ -555,6 +565,8 @@ private bool TryFindMatch(ReadOnlySpan span, ref char searchSpace, Vector5 offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref matchRef) / 2); int lengthRemaining = span.Length - offsetFromStart; + ValidateReadPosition(span, ref matchRef, lengthRemaining); + uint candidateMask = result.GetElementUnsafe(matchOffset); do diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 9bcf088a8dc45..8a968855edb01 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -120,7 +120,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), LoopWithoutRangeCheck: // Read the next input character and either find the next potential match prefix or transition back to the root node. - Debug.Assert(i < span.Length); + Debug.Assert((uint)i < (uint)span.Length); char c = TCaseSensitivity.TransformInput(Unsafe.Add(ref MemoryMarshal.GetReference(span), i)); while (true) @@ -133,6 +133,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), // We were able to extend the current match. If this node contains a potential match, remember that. nodeIndex = childIndex; + Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; if (matchLength != 0) { @@ -228,7 +229,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), LoopWithoutRangeCheck: // Read the next input character and either find the next potential match prefix or transition back to the root node. - Debug.Assert(i < span.Length); + Debug.Assert((uint)i < (uint)span.Length); char c; if (lowSurrogateUpper != LowSurrogateNotSet) { @@ -281,6 +282,7 @@ ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), // We were able to extend the current match. If this node contains a potential match, remember that. nodeIndex = childIndex; + Debug.Assert((uint)nodeIndex < (uint)_nodes.Length); int matchLength = Unsafe.Add(ref nodes, (uint)nodeIndex).MatchLength; if (matchLength != 0) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index 7c5de9c1d0b88..0a7925174d6ea 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -123,6 +123,8 @@ private readonly int IndexOfAnyCore(ReadOnlySpan span) while (true) { + ValidateReadPosition(span, ref current); + if (Unsafe.Add(ref bucketsRef, hash % BucketCount) is string[] bucket) { int startOffset = (int)((nuint)Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref current) / sizeof(char)); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs index e4a2af274784b..e4d43d925d898 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/StringSearchValuesHelper.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Globalization; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics; namespace System.Buffers @@ -12,6 +13,28 @@ namespace System.Buffers // such as normalizing and matching values under different case sensitivity rules. internal static class StringSearchValuesHelper { + [Conditional("DEBUG")] + public static void ValidateReadPosition(ref char searchSpaceStart, int searchSpaceLength, ref char searchSpace, int offset = 0) + { + Debug.Assert(searchSpaceLength >= 0); + + ValidateReadPosition(MemoryMarshal.CreateReadOnlySpan(ref searchSpaceStart, searchSpaceLength), ref searchSpace, offset); + } + + [Conditional("DEBUG")] + public static void ValidateReadPosition(ReadOnlySpan span, ref char searchSpace, int offset = 0) + { + Debug.Assert(offset >= 0); + + nint currentByteOffset = Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref searchSpace); + Debug.Assert(currentByteOffset >= 0); + Debug.Assert((currentByteOffset & 1) == 0); + + int currentOffset = (int)(currentByteOffset / 2); + int availableLength = span.Length - currentOffset; + Debug.Assert(offset <= availableLength); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool StartsWith(ref char matchStart, int lengthRemaining, string[] candidates) where TCaseSensitivity : struct, ICaseSensitivity diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index d80090475c94b..27e1c4267a3ed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -83,6 +83,10 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector512.Count); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector512.Count + (int)(_ch2ByteOffset / 2)); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector512.Count + (int)(_ch3ByteOffset / 2)); + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector512 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); @@ -110,7 +114,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) CandidateFound: // We found potential matches, but they may be false-positives, so we must verify each one. - if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + if (TryMatch(ref searchSpaceStart, searchSpaceLength, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; } @@ -127,6 +131,10 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector256.Count); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector256.Count + (int)(_ch2ByteOffset / 2)); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector256.Count + (int)(_ch3ByteOffset / 2)); + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector256 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); @@ -153,7 +161,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) CandidateFound: // We found potential matches, but they may be false-positives, so we must verify each one. - if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + if (TryMatch(ref searchSpaceStart, searchSpaceLength, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; } @@ -170,6 +178,10 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) while (true) { + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector128.Count); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector128.Count + (int)(_ch2ByteOffset / 2)); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref searchSpace, Vector128.Count + (int)(_ch3ByteOffset / 2)); + // Find which starting positions likely contain a match (likely match all 3 anchor characters). Vector128 result = GetComparisonResult(ref searchSpace, ch2ByteOffset, ch3ByteOffset, ch1, ch2, ch3); @@ -196,7 +208,7 @@ private int IndexOf(ref char searchSpace, int searchSpaceLength) CandidateFound: // We found potential matches, but they may be false-positives, so we must verify each one. - if (TryMatch(ref searchSpaceStart, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) + if (TryMatch(ref searchSpaceStart, searchSpaceLength, ref searchSpace, result.ExtractMostSignificantBits(), out int offset)) { return offset; } @@ -298,7 +310,7 @@ private static Vector512 GetComparisonResult(ref char searchSpace, nuint c } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask, out int offsetFromStart) + private bool TryMatch(ref char searchSpaceStart, int searchSpaceLength, ref char searchSpace, uint mask, out int offsetFromStart) { // 'mask' encodes the input positions where at least 3 characters likely matched. // Verify each one to see if we've found a match, otherwise return back to the vectorized loop. @@ -309,6 +321,8 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref matchRef, _value.Length); + if (TCaseSensitivity.Equals(ref matchRef, _value)) { offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); @@ -324,7 +338,7 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, uint mask } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mask, out int offsetFromStart) + private bool TryMatch(ref char searchSpaceStart, int searchSpaceLength, ref char searchSpace, ulong mask, out int offsetFromStart) { // 'mask' encodes the input positions where at least 3 characters likely matched. // Verify each one to see if we've found a match, otherwise return back to the vectorized loop. @@ -335,6 +349,8 @@ private bool TryMatch(ref char searchSpaceStart, ref char searchSpace, ulong mas ref char matchRef = ref Unsafe.AddByteOffset(ref searchSpace, bitPos); + ValidateReadPosition(ref searchSpaceStart, searchSpaceLength, ref matchRef, _value.Length); + if (TCaseSensitivity.Equals(ref matchRef, _value)) { offsetFromStart = (int)((nuint)Unsafe.ByteOffset(ref searchSpaceStart, ref matchRef) / 2); From ba5a95b20c6199943fcdd02044f206c773be00f7 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Tue, 25 Jul 2023 19:14:57 +0200 Subject: [PATCH 36/43] Add a description of the overall Teddy algorithm --- .../AsciiStringSearchValuesTeddyBase.cs | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index 348a0a0ec2bf8..b00690f8b9df9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -14,12 +14,71 @@ namespace System.Buffers { - // For a description of the Teddy multi-substring matching algorithm, - // see https://github.com/BurntSushi/aho-corasick/blob/master/src/packed/teddy/README.md + // This is an implementation of the "Teddy" vectorized multi-substring matching algorithm. + // + // We have several vectorized string searching approaches implemented as part of SearchValues, among them are: + // - 'IndexOfAnyAsciiSearcher', which can quickly find the next position of any character in a set. + // - 'SingleStringSearchValuesThreeChars', which can determine the likely positions where a value may start. + // The fast scan for starting positions is followed by a verification step that rules out false positives. + // To reduce the number of false positives, the initial scan looks for multiple characters at different positions, + // and only considers candidates where all of those match at the same time. + // + // Teddy combines the two to search for multiple values at the same time. + // Similar to 'SingleStringSearchValuesThreeChars', it employs the starting positions scan and verification steps. + // To reduce the number of values we have to check during verification, it also checks multiple characters in the initial scan. + // We could implement that by just merging the two approaches: check for any of the value characters at position 0, 1, 2, then + // AND those results together and verify potential matches. The issue with this approach is that we would always have to check + // all values in the verification step, and we would be hitting many false positives as the number of values increased. + // + // What is special about Teddy is how we perform that initial scan to not only determine the possible starting locations, + // but also which values are the potential matches at each of those offsets. + // Instead of encoding all starting characters at a given position into a bitmap that can only answer yes/no whether a given + // character is present in the set, we want to encode both the character and the values in which it appears. + // We only have 128* bits to work with, so we do this by encoding 8 bits of information for each nibble (half byte). + // Those 8 bits represent a bitmask of values that contain that nibble at that location. + // If we compare the input against two such bitmaps and AND the results together, we can determine which positions in the input + // contained a matching character, and which of our values matched said character at that position. + // We repeat this a few more times (checking 3 bytes or 6 nibbles for N=3) at different offsets to reduce the number of false positives. + // See 'TeddyBucketizer.GenerateNonBucketizedFingerprint' for details around how such a bitmap is constructed. + // + // For example if we are searching for strings "Teddy" and "Bear", we will look for 'T' or 'B' at position 0, 'e' at position 1, ... + // To look for 'T' (0x54) or 'B' (0x42), we will check for a high nibble of 5 or 4, and lower nibble of 4 or 2. + // Our bitmaps will look like so (bit 1 represents "Teddy", 2 represents "Bear"): + // high: [0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // low: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // ^ ^ ^ ^ + // For an input like "TeddyBearFactory", our result will be + // input: [T, e, d, d, y, B, e, a, r, F, a, c, t, o, r, y] + // high: [1, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] + // low: [1, 0, 1, 1, 0, 2, 0, 0, 2, 0, 0, 0, 1, 0, 2, 0] + // AND: [1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // ^ ^ + // Note how we had quite a few false positives for individual nibbles that we ruled away after checking both nibbles. + // See 'TeddyHelper.ProcessInputN3' for details about how we combine results for multiple characters at different offsets. + // + // The description above states that we can only encode the information about 8 values. To get around that limitation + // we group multiple values together into buckets. Instead of looking for positions where a single value may match, + // we look for positions where any value from a given bucket may match. + // When creating the bitmap we don't set the bit for just one nibble value, but for each of the values in that bucket. + // For example if "Teddy" and "Bear" were both in the same bucket, the high nibble bitmap would map both 5 and 4 to the same bucket. + // We may see more false positives ('R' (0x52) and 'D' (0x44) would now also map to the same bucket), but we get to search for + // many more values at the same time. Instead of 8 values, we are now capable of looking for 8 buckets of values at the same time. + // See 'TeddyBucketizer.Bucketize' for details about how values are grouped into buckets. + // See 'TeddyBucketizer.GenerateBucketizedFingerprint' for details around how such a bitmap is constructed. + // + // To handle case-insensitive matching, all values are normalized to their uppercase equivalents ahead of time and the bitmaps are + // generated as if all characters were uppercase. During the search, the input is also transformed into uppercase before being compared. + // + // * With wider vectors (256- and 512-bit), we have more bits available, but we currently only duplicate the original 128 bits + // and perform the search on more characters at a time. We could instead choose to encode more information per nibble to trade + // the number of characters we check per loop iteration for fewer false positives we then have to rule out during the verification step. + // + // For an alternative description of the algorithm, see + // https://github.com/BurntSushi/aho-corasick/blob/8d735471fc12f0ca570cead8e17342274fae6331/src/packed/teddy/README.md internal abstract class AsciiStringSearchValuesTeddyBase : StringSearchValuesRabinKarp where TBucketized : struct, SearchValues.IRuntimeConst - where TStartCaseSensitivity : struct, ICaseSensitivity - where TCaseSensitivity : struct, ICaseSensitivity + where TStartCaseSensitivity : struct, ICaseSensitivity // Refers to the characters being matched by Teddy + where TCaseSensitivity : struct, ICaseSensitivity // Refers to the rest of the value for the verification step { // We may be using N2 or N3 mode depending on whether we're checking 2 or 3 starting bytes for each bucket. // The result of ProcessInputN2 and ProcessInputN3 are offset by 1 and 2 positions respectively (MatchStartOffsetN2 and MatchStartOffsetN3). From cc7710bc7013fea9e1f6153c447d59ef430ce629 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 26 Jul 2023 03:01:57 +0200 Subject: [PATCH 37/43] Improve the Teddy comment a bit more --- .../AsciiStringSearchValuesTeddyBase.cs | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index b00690f8b9df9..a40f3eecf31b8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -29,6 +29,10 @@ namespace System.Buffers // We could implement that by just merging the two approaches: check for any of the value characters at position 0, 1, 2, then // AND those results together and verify potential matches. The issue with this approach is that we would always have to check // all values in the verification step, and we would be hitting many false positives as the number of values increased. + // For example, if you are searching for "Teddy" and "Bear", position 0 could be either 'T' or 'B', position 1 could be 'e', + // and position 2 could be 'd' or 'a'. We would do separate comparisons for each of those positions and then AND together the result. + // Because there is no correlation between the values, we would get false positives for inputs like "Bed" and "Tea", + // and we wouldn't know whether the match location was because of "Teddy" or "Bear", and thus which to proceed to verify. // // What is special about Teddy is how we perform that initial scan to not only determine the possible starting locations, // but also which values are the potential matches at each of those offsets. @@ -43,16 +47,21 @@ namespace System.Buffers // // For example if we are searching for strings "Teddy" and "Bear", we will look for 'T' or 'B' at position 0, 'e' at position 1, ... // To look for 'T' (0x54) or 'B' (0x42), we will check for a high nibble of 5 or 4, and lower nibble of 4 or 2. - // Our bitmaps will look like so (bit 1 represents "Teddy", 2 represents "Bear"): - // high: [0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - // low: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - // ^ ^ ^ ^ + // Each value's presence is indicated by 1 bit. We will use 1 (0b00000001) for the first value ("Teddy") and 2 (0b00000010) for "Bear". + // Our bitmaps will look like so (1 is set for high 5 and low 4, 2 is set for high 4 and low 2): + // bitmapHigh: [0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // bitmapLow: [0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + // ^ ^ ^ ^ + // + // To map an input nibble to its corresponding bitmask, we use 'Shuffle(bitmap, nibble)'. // For an input like "TeddyBearFactory", our result will be - // input: [T, e, d, d, y, B, e, a, r, F, a, c, t, o, r, y] - // high: [1, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] - // low: [1, 0, 1, 1, 0, 2, 0, 0, 2, 0, 0, 0, 1, 0, 2, 0] - // AND: [1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - // ^ ^ + // input: [T, e, d, d, y, B, e, a, r, F, a, c, t, o, r, y] + // inputHigh: [5, 6, 6, 6, 7, 4, 6, 6, 7, 4, 6, 6, 7, 6, 7, 7] (values in hex) + // inputLow: [4, 5, 4, 4, 9, 2, 5, 1, 2, 6, 1, 3, 4, F, 2, 9] (values in hex) + // resultHigh: [1, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] + // resultLow: [1, 0, 1, 1, 0, 2, 0, 0, 2, 0, 0, 0, 1, 0, 2, 0] + // result: [1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] (resultHigh & resultLow) + // ^ ^ // Note how we had quite a few false positives for individual nibbles that we ruled away after checking both nibbles. // See 'TeddyHelper.ProcessInputN3' for details about how we combine results for multiple characters at different offsets. // @@ -66,6 +75,13 @@ namespace System.Buffers // See 'TeddyBucketizer.Bucketize' for details about how values are grouped into buckets. // See 'TeddyBucketizer.GenerateBucketizedFingerprint' for details around how such a bitmap is constructed. // + // Teddy works in terms of bytes, but .NET chars represent UTF-16 code units. + // We currently only use Teddy if the 2 or 3 starting characters are all ASCII. This limitation could be lifted in the future if needed. + // Since we know that all of the characters we are looking for are ASCII, we also know that only other ASCII characters will match against them. + // Making use of that fact, we narrow UTF-16 code units into bytes when reading the input (see 'TeddyHelper.LoadAndPack16AsciiChars'). + // While such narrowing does corrupt non-ASCII values, they are all mapped to values outside of ASCII, so they won't match anyway. + // ASCII values remain unaffected since their high byte in UTF-16 representation is 0. + // // To handle case-insensitive matching, all values are normalized to their uppercase equivalents ahead of time and the bitmaps are // generated as if all characters were uppercase. During the search, the input is also transformed into uppercase before being compared. // From d4e9557051073c817ac41189a78d1f7cd4b29ae9 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Wed, 26 Jul 2023 03:06:14 +0200 Subject: [PATCH 38/43] Comment space alignment --- .../SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index a40f3eecf31b8..5c1378bbd1bcc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -61,7 +61,7 @@ namespace System.Buffers // resultHigh: [1, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] // resultLow: [1, 0, 1, 1, 0, 2, 0, 0, 2, 0, 0, 0, 1, 0, 2, 0] // result: [1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] (resultHigh & resultLow) - // ^ ^ + // ^ ^ // Note how we had quite a few false positives for individual nibbles that we ruled away after checking both nibbles. // See 'TeddyHelper.ProcessInputN3' for details about how we combine results for multiple characters at different offsets. // From efdd97ae6572d66e8ac0155f2c14141f7abd7196 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 28 Jul 2023 01:09:29 +0200 Subject: [PATCH 39/43] Add a fallback implementation to deal with incomplete surrogate pairs --- .../System.Private.CoreLib.Shared.projitems | 1 + .../System/Globalization/SurrogateCasing.cs | 6 +++ .../Strings/Helpers/AhoCorasick.cs | 3 ++ ...ltiStringIgnoreCaseSearchValuesFallback.cs | 42 +++++++++++++++++++ .../Strings/StringSearchValues.cs | 40 ++++++++++++++++++ 5 files changed, 92 insertions(+) create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/MultiStringIgnoreCaseSearchValuesFallback.cs diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index a59be71ed501b..0f3ea49fa5e20 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -451,6 +451,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/SurrogateCasing.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/SurrogateCasing.cs index 70b5e47be1766..a867a04fbd9e3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/SurrogateCasing.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/SurrogateCasing.cs @@ -16,6 +16,9 @@ internal static void ToUpper(char h, char l, out char hr, out char lr) Debug.Assert(char.IsLowSurrogate(l)); UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(CharUnicodeInfo.ToUpper(UnicodeUtility.GetScalarFromUtf16SurrogatePair(h, l)), out hr, out lr); + + Debug.Assert(char.IsHighSurrogate(hr)); + Debug.Assert(char.IsLowSurrogate(lr)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -25,6 +28,9 @@ internal static void ToLower(char h, char l, out char hr, out char lr) Debug.Assert(char.IsLowSurrogate(l)); UnicodeUtility.GetUtf16SurrogatesFromSupplementaryPlaneScalar(CharUnicodeInfo.ToLower(UnicodeUtility.GetScalarFromUtf16SurrogatePair(h, l)), out hr, out lr); + + Debug.Assert(char.IsHighSurrogate(hr)); + Debug.Assert(char.IsLowSurrogate(lr)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 8a968855edb01..66a5576103729 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -340,6 +340,9 @@ private static void SurrogateToUpperNLS(char h, char l, out char hr, out char lr hr = destination[0]; lr = destination[1]; + + Debug.Assert(char.IsHighSurrogate(hr)); + Debug.Assert(char.IsLowSurrogate(lr)); } public interface IFastScan { } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/MultiStringIgnoreCaseSearchValuesFallback.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/MultiStringIgnoreCaseSearchValuesFallback.cs new file mode 100644 index 0000000000000..a3e45154f442b --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/MultiStringIgnoreCaseSearchValuesFallback.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Buffers +{ + internal sealed class MultiStringIgnoreCaseSearchValuesFallback : StringSearchValuesBase + { + private readonly string[] _values; + + public MultiStringIgnoreCaseSearchValuesFallback(HashSet uniqueValues) : base(uniqueValues) + { + _values = new string[uniqueValues.Count]; + uniqueValues.CopyTo(_values, 0); + } + + /// + /// This method is intentionally implemented in a way that checks haystack positions one at a time. + /// See the description in . + /// + internal override int IndexOfAnyMultiString(ReadOnlySpan span) + { + string[] values = _values; + + for (int i = 0; i < span.Length; i++) + { + ReadOnlySpan remaining = span.Slice(i); + + foreach (string value in values) + { + if (remaining.StartsWith(value, StringComparison.OrdinalIgnoreCase)) + { + return i; + } + } + } + + return -1; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 4e07b3de8e827..4310fc96771c1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -140,6 +140,13 @@ private static SearchValues CreateFromNormalizedValues( if (nonAsciiAffectedByCaseConversion) { + if (ContainsIncompleteSurrogatePairs(values)) + { + // Aho-Corasick can't deal with the matching semantics of standalone surrogate code units. + // We will use a slow but correct O(n * m) fallback implementation. + return new MultiStringIgnoreCaseSearchValuesFallback(uniqueValues); + } + return PickAhoCorasickImplementation(ahoCorasick, uniqueValues); } @@ -370,5 +377,38 @@ private static void AnalyzeValues( } } } + + private static bool ContainsIncompleteSurrogatePairs(ReadOnlySpan values) + { + foreach (string value in values) + { + int i = value.AsSpan().IndexOfAnyInRange(CharUnicodeInfo.HIGH_SURROGATE_START, CharUnicodeInfo.LOW_SURROGATE_END); + if (i < 0) + { + continue; + } + + for (; (uint)i < (uint)value.Length; i++) + { + if (char.IsHighSurrogate(value[i])) + { + if ((uint)(i + 1) >= (uint)value.Length || !char.IsLowSurrogate(value[i + 1])) + { + // High surrogate not followed by a low surrogate. + return true; + } + + i++; + } + else if (char.IsLowSurrogate(value[i])) + { + // Low surrogate not preceded by a high surrogate. + return true; + } + } + } + + return false; + } } } From 1a3e35d4ac8025faaca8cf93c4835327f58ecb1b Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 28 Jul 2023 01:51:01 +0200 Subject: [PATCH 40/43] More edge case tests --- .../tests/Span/StringSearchValues.cs | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index dd3067535b958..015815791faf4 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -147,6 +147,16 @@ void AssertIndexOfAnyAndFriends(Span values, int any, int anyExcept, int [InlineData(StringComparison.OrdinalIgnoreCase, 4, "AAAA\u017FBSBsBBCCCC", "\u017F")] [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAASB\u017FBsBBCCCC", "\u017F")] [InlineData(StringComparison.OrdinalIgnoreCase, 6, "AAAAsB\u017FBSBBCCCC", "\u017F")] + // A few misc non-ASCII examples + [InlineData(StringComparison.OrdinalIgnoreCase, 2, "\0\u1226\u2C5F\0\n\0\u1226\u1242", "hh\u0012\uFE00\u26FF\0\u6C00\u2C00\0b, \u2C5F\0")] + [InlineData(StringComparison.OrdinalIgnoreCase, -1, "barkbarK", "foo, bar\u212A")] + [InlineData(StringComparison.OrdinalIgnoreCase, 4, "bar\u212AbarK", "foo, bark")] + [InlineData(StringComparison.OrdinalIgnoreCase, 0, "bar\u03A3barK", "foo, bar\u03C3")] + [InlineData(StringComparison.OrdinalIgnoreCase, 1, "bar\u03A3barK", "foo, ar\u03C3")] + [InlineData(StringComparison.OrdinalIgnoreCase, 1, " foo\u0131", "foo\u0131")] + [InlineData(StringComparison.OrdinalIgnoreCase, 1, " foo\u0131", "bar, foo\u0131")] + [InlineData(StringComparison.OrdinalIgnoreCase, -1, "fooifooIfoo\u0130", "bar, foo\u0131")] + [InlineData(StringComparison.OrdinalIgnoreCase, -1, "fooifooIfoo\u0131", "bar, foo\u0130")] public static void IndexOfAny(StringComparison comparisonType, int expected, string text, string? values) { Span textSpan = text.ToArray(); // Test non-readonly Span overloads @@ -155,15 +165,55 @@ public static void IndexOfAny(StringComparison comparisonType, int expected, str SearchValues stringValues = SearchValues.Create(valuesArray, comparisonType); + Assert.Equal(expected, IndexOfAnyReferenceImpl(text, valuesArray, comparisonType)); + Assert.Equal(expected, text.AsSpan().IndexOfAny(stringValues)); Assert.Equal(expected, textSpan.IndexOfAny(stringValues)); - Assert.Equal(expected, IndexOfAnyReferenceImpl(text, valuesArray, comparisonType)); - Assert.Equal(expected >= 0, text.AsSpan().ContainsAny(stringValues)); Assert.Equal(expected >= 0, textSpan.ContainsAny(stringValues)); } + [Fact] + public static void IndexOfAny_InvalidUtf16() + { + // Not using [InlineData] to prevent Xunit from modifying the invalid strings. + // These strings have a high surrogate without the full pair. + IndexOfAny(StringComparison.Ordinal, 1, " foo\uD800bar", "foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.Ordinal, -1, " foo\uD801bar", "foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.Ordinal, 2, " foo\uD800bar", "oo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.Ordinal, -1, " foo\uD801bar", "oo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " foo\uD800bar", "foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " foo\uD801bar", "foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 2, " foo\uD800bar", "oo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " foo\uD801bar", "oo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " fOo\uD800bar", "Foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " fOo\uD801bar", "Foo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 2, " foo\uD800bAr", "Oo\uD800bar, bar\uD800foo"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " foO\uD801bar", "oo\uD800baR, bar\uD800foo"); + + // Low surrogate without the high surrogate. + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, "\uD801\uDCD8\uD8FB\uDCD8", "foo, \uDCD8"); + } + + [Fact] + public static void IndexOfAny_CanProduceDifferentResultsUnderNls() + { + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40", "\U00016E60"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40abc", "\U00016E60, abc"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " abc\U00016E40", "abc\U00016E60"); + + if (CanTestNls) + { + RunUsingNLS(static () => + { + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " \U00016E40", "\U00016E60"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 3, " \U00016E40abc", "\U00016E60, abc"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, -1, " abc\U00016E40", "abc\U00016E60"); + }); + } + } + [Fact] public static void Create_OnlyOrdinalComparisonIsSupported() { From d1c69eb48df475ee822cdfd1ed2eb81c5782de36 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 28 Jul 2023 02:05:18 +0200 Subject: [PATCH 41/43] Add some comments about the time complexity of different implementations --- .../SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs | 1 + .../src/System/SearchValues/Strings/Helpers/AhoCorasick.cs | 1 + .../src/System/SearchValues/Strings/Helpers/RabinKarp.cs | 1 + .../SearchValues/Strings/SingleStringSearchValuesThreeChars.cs | 1 + 4 files changed, 4 insertions(+) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs index 5c1378bbd1bcc..e465aae605fb1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/AsciiStringSearchValuesTeddyBase.cs @@ -91,6 +91,7 @@ namespace System.Buffers // // For an alternative description of the algorithm, see // https://github.com/BurntSushi/aho-corasick/blob/8d735471fc12f0ca570cead8e17342274fae6331/src/packed/teddy/README.md + // Has an O(i * m) worst-case, with the expected time closer to O(n) for good bucket distributions. internal abstract class AsciiStringSearchValuesTeddyBase : StringSearchValuesRabinKarp where TBucketized : struct, SearchValues.IRuntimeConst where TStartCaseSensitivity : struct, ICaseSensitivity // Refers to the characters being matched by Teddy diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index 66a5576103729..b193505570eeb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -13,6 +13,7 @@ namespace System.Buffers /// An implementation of the Aho-Corasick algorithm we use as a fallback when we can't use Teddy /// (either due to missing hardware intrinsics, or due to characteristics of the values used). /// https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm + /// Works in O(n). /// internal readonly struct AhoCorasick { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index 0a7925174d6ea..d420c70d16c19 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -13,6 +13,7 @@ namespace System.Buffers /// An implementation of the Rabin-Karp algorithm we use as a fallback for /// short inputs that we can't handle with Teddy. /// https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm + /// Has an O(i * m) worst-case, but we will only use it for very short inputs. /// internal readonly struct RabinKarp { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs index 27e1c4267a3ed..f470eb8d503bd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/SingleStringSearchValuesThreeChars.cs @@ -13,6 +13,7 @@ namespace System.Buffers // Based on SpanHelpers.IndexOf(ref char, int, ref char, int) // This implementation uses 3 precomputed anchor points when searching. // This implementation may also be used for length=2 values, in which case two anchors point at the same position. + // Has an O(i * m) worst-case, with the expected time closer to O(n) for most inputs. internal sealed class SingleStringSearchValuesThreeChars : SearchValues where TCaseSensitivity : struct, ICaseSensitivity { From 177cc7cde5426d28d5ea5c9397585a215e94961c Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 28 Jul 2023 04:32:20 +0200 Subject: [PATCH 42/43] Fix NLS test --- .../System.Memory/tests/Span/StringSearchValues.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index 015815791faf4..57c53d1e55730 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -199,9 +199,15 @@ public static void IndexOfAny_InvalidUtf16() [Fact] public static void IndexOfAny_CanProduceDifferentResultsUnderNls() { - IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40", "\U00016E60"); - IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40abc", "\U00016E60, abc"); - IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " abc\U00016E40", "abc\U00016E60"); + if (CanTestInvariantCulture) + { + RunUsingInvariantCulture(static () => + { + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40", "\U00016E60"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " \U00016E40abc", "\U00016E60, abc"); + IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, " abc\U00016E40", "abc\U00016E60"); + }); + } if (CanTestNls) { From 6c64e45853f80e2d925c9c3cf5955aab3a26f016 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Fri, 4 Aug 2023 23:39:35 +0200 Subject: [PATCH 43/43] React to compiler and ref readonly changes --- .../src/System/SearchValues/Strings/Helpers/AhoCorasick.cs | 4 ++-- .../SearchValues/Strings/Helpers/EightPackedReferences.cs | 4 ++-- .../src/System/SearchValues/Strings/StringSearchValues.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs index b193505570eeb..8bedbe3bb47f3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/AhoCorasick.cs @@ -101,7 +101,7 @@ private readonly int IndexOfAnyCore(ReadOnly int offset = IndexOfAnyAsciiSearcher.IndexOfAnyVectorized( ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), i)), remainingLength, - ref Unsafe.AsRef(_startingCharsAsciiBitmap)); + ref Unsafe.AsRef(in _startingCharsAsciiBitmap)); if (offset < 0) { @@ -210,7 +210,7 @@ private readonly int IndexOfAnyCaseInsensitiveUnicode(ReadOnly int offset = IndexOfAnyAsciiSearcher.IndexOfAnyVectorized( ref Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(span), i)), remainingLength, - ref Unsafe.AsRef(_startingCharsAsciiBitmap)); + ref Unsafe.AsRef(in _startingCharsAsciiBitmap)); if (offset < 0) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs index d667f628c7dc6..b85a7e145c51a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/EightPackedReferences.cs @@ -7,10 +7,10 @@ namespace System.Buffers { [InlineArray(8)] - internal readonly struct EightPackedReferences + internal struct EightPackedReferences { #pragma warning disable CA1823 // Unused field -- https://github.com/dotnet/roslyn-analyzers/issues/6788 - private readonly object? _ref0; + private object? _ref0; #pragma warning restore CA1823 public EightPackedReferences(ReadOnlySpan values) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 4310fc96771c1..77bcdd0ef9607 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -34,7 +34,7 @@ public static SearchValues Create(ReadOnlySpan values, bool igno ArgumentNullException.ThrowIfNull(value, nameof(values)); string normalizedValue = NormalizeIfNeeded(value, ignoreCase); - AnalyzeValues(new ReadOnlySpan(normalizedValue), ref ignoreCase, out bool ascii, out bool asciiLettersOnly, out _, out _); + AnalyzeValues(new ReadOnlySpan(ref normalizedValue), ref ignoreCase, out bool ascii, out bool asciiLettersOnly, out _, out _); return CreateForSingleValue(normalizedValue, uniqueValues: null, ignoreCase, ascii, asciiLettersOnly); }