From dd5fa5bb05348948409a4787138b4af952d58df8 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Tue, 30 Jun 2020 23:30:05 +0800 Subject: [PATCH 01/12] replace DataCache.Find by DataCache.Seek --- src/neo/IO/Caching/CloneCache.cs | 4 +-- src/neo/IO/Caching/DataCache.cs | 32 ++++++++++++++++--- src/neo/IO/Caching/SeekDirection.cs | 8 +++++ src/neo/Persistence/IReadOnlyStore.cs | 3 +- src/neo/Persistence/MemorySnapshot.cs | 7 ++-- src/neo/Persistence/MemoryStore.cs | 7 ++-- src/neo/Persistence/StoreDataCache.cs | 4 +-- .../neo.UnitTests/IO/Caching/UT_DataCache.cs | 4 +-- 8 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 src/neo/IO/Caching/SeekDirection.cs diff --git a/src/neo/IO/Caching/CloneCache.cs b/src/neo/IO/Caching/CloneCache.cs index 559ee3d279..c0e46de929 100644 --- a/src/neo/IO/Caching/CloneCache.cs +++ b/src/neo/IO/Caching/CloneCache.cs @@ -24,9 +24,9 @@ protected override void DeleteInternal(TKey key) innerCache.Delete(key); } - protected override IEnumerable<(TKey, TValue)> FindInternal(byte[] key_prefix) + protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction = SeekDirection.Forward) { - foreach (var (key, value) in innerCache.Find(key_prefix)) + foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction)) yield return (key, value.Clone()); } diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index 4b39b5b7cc..dd21b024b0 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -149,14 +149,14 @@ public void Delete(TKey key) /// /// Must maintain the deserialized format of TKey /// Entries found with the desired prefix - public IEnumerable<(TKey Key, TValue Value)> Find(byte[] key_prefix = null) + public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { IEnumerable<(byte[], TKey, TValue)> cached; HashSet cachedKeySet; lock (dictionary) { cached = dictionary - .Where(p => p.Value.State != TrackState.Deleted && (key_prefix == null || p.Key.ToArray().AsSpan().StartsWith(key_prefix))) + .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || ByteArrayComparer.Default.Compare(p.Key.ToArray(), keyOrPrefix) * Convert.ToSByte(direction) >= 0)) .Select(p => ( KeyBytes: p.Key.ToArray(), @@ -167,7 +167,7 @@ public void Delete(TKey key) .ToArray(); cachedKeySet = new HashSet(dictionary.Keys); } - var uncached = FindInternal(key_prefix ?? Array.Empty()) + var uncached = SeekInternal(keyOrPrefix ?? Array.Empty(), direction) .Where(p => !cachedKeySet.Contains(p.Key)) .Select(p => ( @@ -185,7 +185,7 @@ public void Delete(TKey key) i2 = c2 ? e2.Current : default; while (c1 || c2) { - if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) + if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) * Convert.ToSByte(direction) < 0)) { yield return (i1.Key, i1.Item); c1 = e1.MoveNext(); @@ -201,7 +201,29 @@ public void Delete(TKey key) } } - protected abstract IEnumerable<(TKey Key, TValue Value)> FindInternal(byte[] key_prefix); + /// + /// Find the entries that start with the `key_prefix` + /// + /// Must maintain the deserialized format of TKey + /// Entries found with the desired prefix + public IEnumerable<(TKey Key, TValue Value)> Find(byte[] key_prefix = null) + { + var enumerator = Seek(key_prefix, SeekDirection.Forward).GetEnumerator(); + while (enumerator.MoveNext()) + if (enumerator.Current.Key.ToArray().AsSpan().StartsWith(key_prefix)) + yield return enumerator.Current; + } + + public IEnumerable<(TKey Key, TValue Value)> FindRange(TKey start, TKey end) + { + var enumerator = Seek(start.ToArray(), SeekDirection.Forward).GetEnumerator(); + var endKey = end?.ToArray(); + while (enumerator.MoveNext()) + if (endKey is null || ByteArrayComparer.Default.Compare(enumerator.Current.Key.ToArray(), endKey) < 0) + yield return enumerator.Current; + } + + protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward); public IEnumerable GetChangeSet() { diff --git a/src/neo/IO/Caching/SeekDirection.cs b/src/neo/IO/Caching/SeekDirection.cs new file mode 100644 index 0000000000..5387fd8311 --- /dev/null +++ b/src/neo/IO/Caching/SeekDirection.cs @@ -0,0 +1,8 @@ +namespace Neo.IO.Caching +{ + public enum SeekDirection : sbyte + { + Forward = 1, + Backward = -1 + } +} diff --git a/src/neo/Persistence/IReadOnlyStore.cs b/src/neo/Persistence/IReadOnlyStore.cs index 7a23bd4c80..234a36f534 100644 --- a/src/neo/Persistence/IReadOnlyStore.cs +++ b/src/neo/Persistence/IReadOnlyStore.cs @@ -1,3 +1,4 @@ +using Neo.IO.Caching; using System.Collections.Generic; namespace Neo.Persistence @@ -7,7 +8,7 @@ namespace Neo.Persistence /// public interface IReadOnlyStore { - IEnumerable<(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix); + IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] key, SeekDirection direction); byte[] TryGet(byte table, byte[] key); } } diff --git a/src/neo/Persistence/MemorySnapshot.cs b/src/neo/Persistence/MemorySnapshot.cs index 5b1dc35742..6d2c575f49 100644 --- a/src/neo/Persistence/MemorySnapshot.cs +++ b/src/neo/Persistence/MemorySnapshot.cs @@ -1,4 +1,5 @@ using Neo.IO; +using Neo.IO.Caching; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -41,11 +42,11 @@ public void Dispose() { } - public IEnumerable<(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix) + public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) { IEnumerable> records = immutableData[table]; - if (prefix?.Length > 0) - records = records.Where(p => p.Key.AsSpan().StartsWith(prefix)); + if (keyOrPrefix?.Length > 0) + records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); records = records.OrderBy(p => p.Key, ByteArrayComparer.Default); return records.Select(p => (p.Key, p.Value)); } diff --git a/src/neo/Persistence/MemoryStore.cs b/src/neo/Persistence/MemoryStore.cs index 5b6c09c58f..23f09b819f 100644 --- a/src/neo/Persistence/MemoryStore.cs +++ b/src/neo/Persistence/MemoryStore.cs @@ -1,4 +1,5 @@ using Neo.IO; +using Neo.IO.Caching; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -26,11 +27,11 @@ public void Dispose() { } - public IEnumerable<(byte[] Key, byte[] Value)> Find(byte table, byte[] prefix) + public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) { IEnumerable> records = innerData[table]; - if (prefix?.Length > 0) - records = records.Where(p => p.Key.AsSpan().StartsWith(prefix)); + if (keyOrPrefix?.Length > 0) + records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); records = records.OrderBy(p => p.Key, ByteArrayComparer.Default); foreach (var pair in records) yield return (pair.Key, pair.Value); diff --git a/src/neo/Persistence/StoreDataCache.cs b/src/neo/Persistence/StoreDataCache.cs index 995a34290f..01ac06d2ed 100644 --- a/src/neo/Persistence/StoreDataCache.cs +++ b/src/neo/Persistence/StoreDataCache.cs @@ -31,9 +31,9 @@ protected override void DeleteInternal(TKey key) snapshot?.Delete(prefix, key.ToArray()); } - protected override IEnumerable<(TKey, TValue)> FindInternal(byte[] key_prefix) + protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) { - return store.Find(prefix, key_prefix).Select(p => (p.Key.AsSerializable(), p.Value.AsSerializable())); + return store.Seek(prefix, keyOrPrefix, direction).Select(p => (p.Key.AsSerializable(), p.Value.AsSerializable())); } protected override TValue GetInternal(TKey key) diff --git a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs index 2b07c8fa0c..492428ee4c 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs @@ -115,9 +115,9 @@ protected override void AddInternal(TKey key, TValue value) InnerDict.Add(key, value); } - protected override IEnumerable<(TKey, TValue)> FindInternal(byte[] key_prefix) + protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) { - return InnerDict.Where(kvp => kvp.Key.ToArray().Take(key_prefix.Length).SequenceEqual(key_prefix)).Select(p => (p.Key, p.Value)); + return InnerDict.Where(kvp => ByteArrayComparer.Default.Compare(kvp.Key.ToArray(), keyOrPrefix) * Convert.ToSByte(direction) >= 0).Select(p => (p.Key, p.Value)); } protected override TValue GetInternal(TKey key) From 4a206de22f2b266e5ced9d4ea28be92d55c39012 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Wed, 1 Jul 2020 09:24:30 +0800 Subject: [PATCH 02/12] fix order --- src/neo/IO/ByteArrayComparer.cs | 14 +++++-- src/neo/IO/Caching/DataCache.cs | 2 +- src/neo/Persistence/MemorySnapshot.cs | 2 +- src/neo/Persistence/MemoryStore.cs | 2 +- .../neo.UnitTests/IO/Caching/UT_DataCache.cs | 37 +++++++++++++++++++ .../neo.UnitTests/IO/UT_ByteArrayComparer.cs | 12 +++++- 6 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/neo/IO/ByteArrayComparer.cs b/src/neo/IO/ByteArrayComparer.cs index 478c6e3c7d..733de306f9 100644 --- a/src/neo/IO/ByteArrayComparer.cs +++ b/src/neo/IO/ByteArrayComparer.cs @@ -5,7 +5,15 @@ namespace Neo.IO { internal class ByteArrayComparer : IComparer { - public static readonly ByteArrayComparer Default = new ByteArrayComparer(); + public static readonly ByteArrayComparer Default = new ByteArrayComparer(1); + public static readonly ByteArrayComparer Reverse = new ByteArrayComparer(-1); + + private int Direction; + + private ByteArrayComparer(int direction) + { + Direction = direction; + } public int Compare(byte[] x, byte[] y) { @@ -13,9 +21,9 @@ public int Compare(byte[] x, byte[] y) for (int i = 0; i < length; i++) { int r = x[i].CompareTo(y[i]); - if (r != 0) return r; + if (r != 0) return r * Direction; } - return x.Length.CompareTo(y.Length); + return x.Length.CompareTo(y.Length) * Direction; } } } diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index dd21b024b0..d677497155 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -163,7 +163,7 @@ public void Delete(TKey key) p.Key, p.Value.Item )) - .OrderBy(p => p.KeyBytes, ByteArrayComparer.Default) + .OrderBy(p => p.KeyBytes, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse) .ToArray(); cachedKeySet = new HashSet(dictionary.Keys); } diff --git a/src/neo/Persistence/MemorySnapshot.cs b/src/neo/Persistence/MemorySnapshot.cs index 6d2c575f49..cf619db12f 100644 --- a/src/neo/Persistence/MemorySnapshot.cs +++ b/src/neo/Persistence/MemorySnapshot.cs @@ -47,7 +47,7 @@ public void Dispose() IEnumerable> records = immutableData[table]; if (keyOrPrefix?.Length > 0) records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); - records = records.OrderBy(p => p.Key, ByteArrayComparer.Default); + records = records.OrderBy(p => p.Key, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse); return records.Select(p => (p.Key, p.Value)); } diff --git a/src/neo/Persistence/MemoryStore.cs b/src/neo/Persistence/MemoryStore.cs index 23f09b819f..e9edcefda1 100644 --- a/src/neo/Persistence/MemoryStore.cs +++ b/src/neo/Persistence/MemoryStore.cs @@ -32,7 +32,7 @@ public void Dispose() IEnumerable> records = innerData[table]; if (keyOrPrefix?.Length > 0) records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); - records = records.OrderBy(p => p.Key, ByteArrayComparer.Default); + records = records.OrderBy(p => p.Key, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse); foreach (var pair in records) yield return (pair.Key, pair.Value); } diff --git a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs index 492428ee4c..822054551d 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs @@ -264,6 +264,43 @@ public void TestFind() items.Count().Should().Be(0); } + [TestMethod] + public void TestSeek() + { + myDataCache.Add(new MyKey("key1"), new MyValue("value1")); + myDataCache.Add(new MyKey("key2"), new MyValue("value2")); + + myDataCache.InnerDict.Add(new MyKey("key3"), new MyValue("value3")); + myDataCache.InnerDict.Add(new MyKey("key4"), new MyValue("value4")); + + var items = myDataCache.Seek(new MyKey("key3").ToArray(), SeekDirection.Backward).ToArray(); + items[0].Key.Should().Be(new MyKey("key3")); + items[0].Value.Should().Be(new MyValue("value3")); + items[1].Key.Should().Be(new MyKey("key2")); + items[1].Value.Should().Be(new MyValue("value2")); + items.Count().Should().Be(3); + + items = myDataCache.Seek(new MyKey("key5").ToArray(), SeekDirection.Forward).ToArray(); + items.Count().Should().Be(0); + } + + [TestMethod] + public void TestFindRange() + { + myDataCache.Add(new MyKey("key1"), new MyValue("value1")); + myDataCache.Add(new MyKey("key2"), new MyValue("value2")); + + myDataCache.InnerDict.Add(new MyKey("key3"), new MyValue("value3")); + myDataCache.InnerDict.Add(new MyKey("key4"), new MyValue("value4")); + + var items = myDataCache.FindRange(new MyKey("key3"), new MyKey("key5")).ToArray(); + items[0].Key.Should().Be(new MyKey("key3")); + items[0].Value.Should().Be(new MyValue("value3")); + items[1].Key.Should().Be(new MyKey("key4")); + items[1].Value.Should().Be(new MyValue("value4")); + items.Count().Should().Be(2); + } + [TestMethod] public void TestGetChangeSet() { diff --git a/tests/neo.UnitTests/IO/UT_ByteArrayComparer.cs b/tests/neo.UnitTests/IO/UT_ByteArrayComparer.cs index 562e2b6f6e..89c0c621d7 100644 --- a/tests/neo.UnitTests/IO/UT_ByteArrayComparer.cs +++ b/tests/neo.UnitTests/IO/UT_ByteArrayComparer.cs @@ -10,7 +10,7 @@ public class UT_ByteArrayComparer [TestMethod] public void TestCompare() { - ByteArrayComparer comparer = new ByteArrayComparer(); + ByteArrayComparer comparer = ByteArrayComparer.Default; byte[] x = new byte[0], y = new byte[0]; comparer.Compare(x, y).Should().Be(0); @@ -22,6 +22,16 @@ public void TestCompare() x = new byte[] { 1 }; y = new byte[] { 2 }; comparer.Compare(x, y).Should().Be(-1); + + comparer = ByteArrayComparer.Reverse; + x = new byte[] { 3 }; + comparer.Compare(x, y).Should().Be(-1); + y = x; + comparer.Compare(x, y).Should().Be(0); + + x = new byte[] { 1 }; + y = new byte[] { 2 }; + comparer.Compare(x, y).Should().Be(1); } } } From d5eb5148af5a5f60fda94b297d144e5dc5e300e3 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Wed, 1 Jul 2020 09:31:15 +0800 Subject: [PATCH 03/12] optimize --- src/neo/IO/Caching/DataCache.cs | 7 ++++--- src/neo/Persistence/MemorySnapshot.cs | 5 +++-- src/neo/Persistence/MemoryStore.cs | 5 +++-- tests/neo.UnitTests/IO/Caching/UT_DataCache.cs | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index d677497155..005614c176 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -153,17 +153,18 @@ public void Delete(TKey key) { IEnumerable<(byte[], TKey, TValue)> cached; HashSet cachedKeySet; + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; lock (dictionary) { cached = dictionary - .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || ByteArrayComparer.Default.Compare(p.Key.ToArray(), keyOrPrefix) * Convert.ToSByte(direction) >= 0)) + .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || comparer.Compare(p.Key.ToArray(), keyOrPrefix) >= 0)) .Select(p => ( KeyBytes: p.Key.ToArray(), p.Key, p.Value.Item )) - .OrderBy(p => p.KeyBytes, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse) + .OrderBy(p => p.KeyBytes, comparer) .ToArray(); cachedKeySet = new HashSet(dictionary.Keys); } @@ -185,7 +186,7 @@ public void Delete(TKey key) i2 = c2 ? e2.Current : default; while (c1 || c2) { - if (!c2 || (c1 && ByteArrayComparer.Default.Compare(i1.KeyBytes, i2.KeyBytes) * Convert.ToSByte(direction) < 0)) + if (!c2 || (c1 && comparer.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) { yield return (i1.Key, i1.Item); c1 = e1.MoveNext(); diff --git a/src/neo/Persistence/MemorySnapshot.cs b/src/neo/Persistence/MemorySnapshot.cs index cf619db12f..bf031e18dc 100644 --- a/src/neo/Persistence/MemorySnapshot.cs +++ b/src/neo/Persistence/MemorySnapshot.cs @@ -44,10 +44,11 @@ public void Dispose() public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) { + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; IEnumerable> records = immutableData[table]; if (keyOrPrefix?.Length > 0) - records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); - records = records.OrderBy(p => p.Key, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse); + records = records.Where(p => comparer.Compare(p.Key, keyOrPrefix) >= 0); + records = records.OrderBy(p => p.Key, comparer); return records.Select(p => (p.Key, p.Value)); } diff --git a/src/neo/Persistence/MemoryStore.cs b/src/neo/Persistence/MemoryStore.cs index e9edcefda1..750c396e4a 100644 --- a/src/neo/Persistence/MemoryStore.cs +++ b/src/neo/Persistence/MemoryStore.cs @@ -29,10 +29,11 @@ public void Dispose() public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) { + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; IEnumerable> records = innerData[table]; if (keyOrPrefix?.Length > 0) - records = records.Where(p => ByteArrayComparer.Default.Compare(p.Key, keyOrPrefix) * Convert.ToSByte(direction) >= 0); - records = records.OrderBy(p => p.Key, direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse); + records = records.Where(p => comparer.Compare(p.Key, keyOrPrefix) >= 0); + records = records.OrderBy(p => p.Key, comparer); foreach (var pair in records) yield return (pair.Key, pair.Value); } diff --git a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs index 822054551d..74f3213f82 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_DataCache.cs @@ -117,7 +117,8 @@ protected override void AddInternal(TKey key, TValue value) protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) { - return InnerDict.Where(kvp => ByteArrayComparer.Default.Compare(kvp.Key.ToArray(), keyOrPrefix) * Convert.ToSByte(direction) >= 0).Select(p => (p.Key, p.Value)); + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; + return InnerDict.Where(kvp => comparer.Compare(kvp.Key.ToArray(), keyOrPrefix) >= 0).Select(p => (p.Key, p.Value)); } protected override TValue GetInternal(TKey key) From fa83632dc379b5385f6a045fe5050ebf831a5797 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 Jul 2020 15:25:58 +0800 Subject: [PATCH 04/12] Update ByteArrayComparer.cs --- src/neo/IO/ByteArrayComparer.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/neo/IO/ByteArrayComparer.cs b/src/neo/IO/ByteArrayComparer.cs index 733de306f9..7056ecde3f 100644 --- a/src/neo/IO/ByteArrayComparer.cs +++ b/src/neo/IO/ByteArrayComparer.cs @@ -8,22 +8,26 @@ internal class ByteArrayComparer : IComparer public static readonly ByteArrayComparer Default = new ByteArrayComparer(1); public static readonly ByteArrayComparer Reverse = new ByteArrayComparer(-1); - private int Direction; + private readonly int direction; private ByteArrayComparer(int direction) { - Direction = direction; + this.direction = direction; } public int Compare(byte[] x, byte[] y) { + int r; int length = Math.Min(x.Length, y.Length); for (int i = 0; i < length; i++) { - int r = x[i].CompareTo(y[i]); - if (r != 0) return r * Direction; + r = x[i].CompareTo(y[i]); + if (direction == -1) r = -r; + if (r != 0) return r; } - return x.Length.CompareTo(y.Length) * Direction; + r = x.Length.CompareTo(y.Length); + if (direction == -1) r = -r; + return r; } } } From b578fa7fe2fb3b897ec31829affe149cbca941a5 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 Jul 2020 15:31:35 +0800 Subject: [PATCH 05/12] Update ByteArrayComparer.cs --- src/neo/IO/ByteArrayComparer.cs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/neo/IO/ByteArrayComparer.cs b/src/neo/IO/ByteArrayComparer.cs index 7056ecde3f..8e9b2573c0 100644 --- a/src/neo/IO/ByteArrayComparer.cs +++ b/src/neo/IO/ByteArrayComparer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace Neo.IO { @@ -17,17 +18,21 @@ private ByteArrayComparer(int direction) public int Compare(byte[] x, byte[] y) { - int r; + return direction > 0 + ? CompareInternal(x, y) + : -CompareInternal(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int CompareInternal(byte[] x, byte[] y) + { int length = Math.Min(x.Length, y.Length); for (int i = 0; i < length; i++) { - r = x[i].CompareTo(y[i]); - if (direction == -1) r = -r; + int r = x[i].CompareTo(y[i]); if (r != 0) return r; } - r = x.Length.CompareTo(y.Length); - if (direction == -1) r = -r; - return r; + return x.Length.CompareTo(y.Length); } } } From 966d6de1136a4927dacbce083b4158191574f310 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 1 Jul 2020 09:58:32 +0200 Subject: [PATCH 06/12] Update DataCache.cs --- src/neo/IO/Caching/DataCache.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index 005614c176..866b56f81e 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -145,9 +145,10 @@ public void Delete(TKey key) protected abstract void DeleteInternal(TKey key); /// - /// Find the entries that start with the `key_prefix` + /// Find the entries that start with the `keyOrPrefix` /// - /// Must maintain the deserialized format of TKey + /// Must maintain the deserialized format of TKey + /// Forward or Backward /// Entries found with the desired prefix public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { From f45c52bd7f528f6421b15f848f5fc70f521d03c8 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Wed, 1 Jul 2020 16:13:09 +0800 Subject: [PATCH 07/12] fix comments --- src/neo/IO/Caching/DataCache.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index 005614c176..1362fe162d 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -145,10 +145,11 @@ public void Delete(TKey key) protected abstract void DeleteInternal(TKey key); /// - /// Find the entries that start with the `key_prefix` + /// Find entries that are not less/more than the `keyOrPrefix` /// - /// Must maintain the deserialized format of TKey - /// Entries found with the desired prefix + /// Must maintain the deserialized format of TKey + /// SeekDirection.Forward means not less than `keyOrPrefix`, SeekDirection.Backward means not more than `keyOrPrefix` + /// Entries that are not less/more than the `keyOrPrefix` public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { IEnumerable<(byte[], TKey, TValue)> cached; From ecb2b0d91c068c002031ebd0f9ca9afee34b1862 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Wed, 1 Jul 2020 17:38:59 +0800 Subject: [PATCH 08/12] fix comments --- src/neo/IO/Caching/DataCache.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index 866b56f81e..a62c1a470d 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -145,11 +145,11 @@ public void Delete(TKey key) protected abstract void DeleteInternal(TKey key); /// - /// Find the entries that start with the `keyOrPrefix` + /// Find the entries that not less/more than the `keyOrPrefix` /// /// Must maintain the deserialized format of TKey - /// Forward or Backward - /// Entries found with the desired prefix + /// Forward means not less than, Backward means not more than + /// Entries that are not less/more than the `keyOrPrefix` public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { IEnumerable<(byte[], TKey, TValue)> cached; From 6268a4aa2ddd512d7a26da331179a2e0e2edd61a Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Wed, 1 Jul 2020 18:19:47 +0800 Subject: [PATCH 09/12] fix comments --- src/neo/IO/Caching/DataCache.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index a62c1a470d..9f09d2b44f 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -145,11 +145,11 @@ public void Delete(TKey key) protected abstract void DeleteInternal(TKey key); /// - /// Find the entries that not less/more than the `keyOrPrefix` + /// Seek to the entry with specific key /// - /// Must maintain the deserialized format of TKey - /// Forward means not less than, Backward means not more than - /// Entries that are not less/more than the `keyOrPrefix` + /// The key to be sought + /// The direction of seek + /// An enumerator containing all the entries after seeking. public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { IEnumerable<(byte[], TKey, TValue)> cached; From d0a1820dd44ac4d66b3723543ce26de39abb7dd6 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 Jul 2020 18:35:38 +0800 Subject: [PATCH 10/12] Update DataCache.cs --- src/neo/IO/Caching/DataCache.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index 9f09d2b44f..dc1f5b36ae 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -210,10 +210,9 @@ public void Delete(TKey key) /// Entries found with the desired prefix public IEnumerable<(TKey Key, TValue Value)> Find(byte[] key_prefix = null) { - var enumerator = Seek(key_prefix, SeekDirection.Forward).GetEnumerator(); - while (enumerator.MoveNext()) - if (enumerator.Current.Key.ToArray().AsSpan().StartsWith(key_prefix)) - yield return enumerator.Current; + foreach (var (key, value) in Seek(key_prefix, SeekDirection.Forward)) + if (key.ToArray().AsSpan().StartsWith(key_prefix)) + yield return (key, value); } public IEnumerable<(TKey Key, TValue Value)> FindRange(TKey start, TKey end) From 21f80b3606c55ff1685290122b5bb3cfe163acb7 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 Jul 2020 18:39:05 +0800 Subject: [PATCH 11/12] Update DataCache.cs --- src/neo/IO/Caching/DataCache.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index dc1f5b36ae..e5314967a8 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -217,11 +217,10 @@ public void Delete(TKey key) public IEnumerable<(TKey Key, TValue Value)> FindRange(TKey start, TKey end) { - var enumerator = Seek(start.ToArray(), SeekDirection.Forward).GetEnumerator(); - var endKey = end?.ToArray(); - while (enumerator.MoveNext()) - if (endKey is null || ByteArrayComparer.Default.Compare(enumerator.Current.Key.ToArray(), endKey) < 0) - yield return enumerator.Current; + var endKey = end.ToArray(); + foreach (var (key, value) in Seek(start.ToArray(), SeekDirection.Forward)) + if (ByteArrayComparer.Default.Compare(key.ToArray(), endKey) < 0) + yield return (key, value); } protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward); From 3c2de938b729898c47d99fc14122612598837d8c Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 1 Jul 2020 19:04:01 +0800 Subject: [PATCH 12/12] Reorder methods --- src/neo/IO/Caching/CloneCache.cs | 10 +-- src/neo/IO/Caching/DataCache.cs | 122 +++++++++++++------------- src/neo/Persistence/MemorySnapshot.cs | 12 +-- src/neo/Persistence/MemoryStore.cs | 22 ++--- src/neo/Persistence/StoreDataCache.cs | 8 +- 5 files changed, 87 insertions(+), 87 deletions(-) diff --git a/src/neo/IO/Caching/CloneCache.cs b/src/neo/IO/Caching/CloneCache.cs index c0e46de929..93e01e115a 100644 --- a/src/neo/IO/Caching/CloneCache.cs +++ b/src/neo/IO/Caching/CloneCache.cs @@ -24,15 +24,15 @@ protected override void DeleteInternal(TKey key) innerCache.Delete(key); } - protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction = SeekDirection.Forward) + protected override TValue GetInternal(TKey key) { - foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction)) - yield return (key, value.Clone()); + return innerCache[key].Clone(); } - protected override TValue GetInternal(TKey key) + protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction) { - return innerCache[key].Clone(); + foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction)) + yield return (key, value.Clone()); } protected override TValue TryGetInternal(TKey key) diff --git a/src/neo/IO/Caching/DataCache.cs b/src/neo/IO/Caching/DataCache.cs index e5314967a8..21e4f95bb4 100644 --- a/src/neo/IO/Caching/DataCache.cs +++ b/src/neo/IO/Caching/DataCache.cs @@ -144,65 +144,6 @@ public void Delete(TKey key) protected abstract void DeleteInternal(TKey key); - /// - /// Seek to the entry with specific key - /// - /// The key to be sought - /// The direction of seek - /// An enumerator containing all the entries after seeking. - public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) - { - IEnumerable<(byte[], TKey, TValue)> cached; - HashSet cachedKeySet; - ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; - lock (dictionary) - { - cached = dictionary - .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || comparer.Compare(p.Key.ToArray(), keyOrPrefix) >= 0)) - .Select(p => - ( - KeyBytes: p.Key.ToArray(), - p.Key, - p.Value.Item - )) - .OrderBy(p => p.KeyBytes, comparer) - .ToArray(); - cachedKeySet = new HashSet(dictionary.Keys); - } - var uncached = SeekInternal(keyOrPrefix ?? Array.Empty(), direction) - .Where(p => !cachedKeySet.Contains(p.Key)) - .Select(p => - ( - KeyBytes: p.Key.ToArray(), - p.Key, - p.Value - )); - using (var e1 = cached.GetEnumerator()) - using (var e2 = uncached.GetEnumerator()) - { - (byte[] KeyBytes, TKey Key, TValue Item) i1, i2; - bool c1 = e1.MoveNext(); - bool c2 = e2.MoveNext(); - i1 = c1 ? e1.Current : default; - i2 = c2 ? e2.Current : default; - while (c1 || c2) - { - if (!c2 || (c1 && comparer.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) - { - yield return (i1.Key, i1.Item); - c1 = e1.MoveNext(); - i1 = c1 ? e1.Current : default; - } - else - { - yield return (i2.Key, i2.Item); - c2 = e2.MoveNext(); - i2 = c2 ? e2.Current : default; - } - } - } - } - /// /// Find the entries that start with the `key_prefix` /// @@ -223,8 +164,6 @@ public void Delete(TKey key) yield return (key, value); } - protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward); - public IEnumerable GetChangeSet() { lock (dictionary) @@ -321,6 +260,67 @@ public TValue GetOrAdd(TKey key, Func factory) } } + /// + /// Seek to the entry with specific key + /// + /// The key to be sought + /// The direction of seek + /// An enumerator containing all the entries after seeking. + public IEnumerable<(TKey Key, TValue Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) + { + IEnumerable<(byte[], TKey, TValue)> cached; + HashSet cachedKeySet; + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; + lock (dictionary) + { + cached = dictionary + .Where(p => p.Value.State != TrackState.Deleted && (keyOrPrefix == null || comparer.Compare(p.Key.ToArray(), keyOrPrefix) >= 0)) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value.Item + )) + .OrderBy(p => p.KeyBytes, comparer) + .ToArray(); + cachedKeySet = new HashSet(dictionary.Keys); + } + var uncached = SeekInternal(keyOrPrefix ?? Array.Empty(), direction) + .Where(p => !cachedKeySet.Contains(p.Key)) + .Select(p => + ( + KeyBytes: p.Key.ToArray(), + p.Key, + p.Value + )); + using (var e1 = cached.GetEnumerator()) + using (var e2 = uncached.GetEnumerator()) + { + (byte[] KeyBytes, TKey Key, TValue Item) i1, i2; + bool c1 = e1.MoveNext(); + bool c2 = e2.MoveNext(); + i1 = c1 ? e1.Current : default; + i2 = c2 ? e2.Current : default; + while (c1 || c2) + { + if (!c2 || (c1 && comparer.Compare(i1.KeyBytes, i2.KeyBytes) < 0)) + { + yield return (i1.Key, i1.Item); + c1 = e1.MoveNext(); + i1 = c1 ? e1.Current : default; + } + else + { + yield return (i2.Key, i2.Item); + c2 = e2.MoveNext(); + i2 = c2 ? e2.Current : default; + } + } + } + } + + protected abstract IEnumerable<(TKey Key, TValue Value)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction); + public TValue TryGet(TKey key) { lock (dictionary) diff --git a/src/neo/Persistence/MemorySnapshot.cs b/src/neo/Persistence/MemorySnapshot.cs index bf031e18dc..1474b26df7 100644 --- a/src/neo/Persistence/MemorySnapshot.cs +++ b/src/neo/Persistence/MemorySnapshot.cs @@ -42,7 +42,12 @@ public void Dispose() { } - public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) + public void Put(byte table, byte[] key, byte[] value) + { + writeBatch[table][key.EnsureNotNull()] = value; + } + + public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) { ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; IEnumerable> records = immutableData[table]; @@ -52,11 +57,6 @@ public void Dispose() return records.Select(p => (p.Key, p.Value)); } - public void Put(byte table, byte[] key, byte[] value) - { - writeBatch[table][key.EnsureNotNull()] = value; - } - public byte[] TryGet(byte table, byte[] key) { immutableData[table].TryGetValue(key.EnsureNotNull(), out byte[] value); diff --git a/src/neo/Persistence/MemoryStore.cs b/src/neo/Persistence/MemoryStore.cs index 750c396e4a..67bdd04152 100644 --- a/src/neo/Persistence/MemoryStore.cs +++ b/src/neo/Persistence/MemoryStore.cs @@ -27,17 +27,6 @@ public void Dispose() { } - public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction) - { - ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; - IEnumerable> records = innerData[table]; - if (keyOrPrefix?.Length > 0) - records = records.Where(p => comparer.Compare(p.Key, keyOrPrefix) >= 0); - records = records.OrderBy(p => p.Key, comparer); - foreach (var pair in records) - yield return (pair.Key, pair.Value); - } - public ISnapshot GetSnapshot() { return new MemorySnapshot(innerData); @@ -48,6 +37,17 @@ public void Put(byte table, byte[] key, byte[] value) innerData[table][key.EnsureNotNull()] = value; } + public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte table, byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) + { + ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; + IEnumerable> records = innerData[table]; + if (keyOrPrefix?.Length > 0) + records = records.Where(p => comparer.Compare(p.Key, keyOrPrefix) >= 0); + records = records.OrderBy(p => p.Key, comparer); + foreach (var pair in records) + yield return (pair.Key, pair.Value); + } + public byte[] TryGet(byte table, byte[] key) { innerData[table].TryGetValue(key.EnsureNotNull(), out byte[] value); diff --git a/src/neo/Persistence/StoreDataCache.cs b/src/neo/Persistence/StoreDataCache.cs index 01ac06d2ed..831c88dce1 100644 --- a/src/neo/Persistence/StoreDataCache.cs +++ b/src/neo/Persistence/StoreDataCache.cs @@ -31,14 +31,14 @@ protected override void DeleteInternal(TKey key) snapshot?.Delete(prefix, key.ToArray()); } - protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) + protected override TValue GetInternal(TKey key) { - return store.Seek(prefix, keyOrPrefix, direction).Select(p => (p.Key.AsSerializable(), p.Value.AsSerializable())); + return store.TryGet(prefix, key.ToArray()).AsSerializable(); } - protected override TValue GetInternal(TKey key) + protected override IEnumerable<(TKey, TValue)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction) { - return store.TryGet(prefix, key.ToArray()).AsSerializable(); + return store.Seek(prefix, keyOrPrefix, direction).Select(p => (p.Key.AsSerializable(), p.Value.AsSerializable())); } protected override TValue TryGetInternal(TKey key)