From c7669d813b06d7952f5b61a43c891b589e17c58a Mon Sep 17 00:00:00 2001 From: Panayiotis Savva Date: Thu, 14 Mar 2024 20:30:17 +0200 Subject: [PATCH 1/3] Upgrade LiteDB@5.0.14 --- src/Blockcore/Blockcore.csproj | 2 +- .../AddressIndexing/AddressIndexRepository.cs | 2 +- .../AddressIndexing/AddressIndexer.cs | 7 +++--- .../AddressIndexerOutpointsRepository.cs | 8 ++++--- .../AddressIndexerOutpointsRepositoryTests.cs | 4 ++-- .../AddressIndexerTests.cs | 24 +++++++++---------- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/Blockcore/Blockcore.csproj b/src/Blockcore/Blockcore.csproj index 987d5a493..7837e0441 100644 --- a/src/Blockcore/Blockcore.csproj +++ b/src/Blockcore/Blockcore.csproj @@ -24,7 +24,7 @@ - + diff --git a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs index 9b56ad8b7..97ca4efaf 100644 --- a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs +++ b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs @@ -13,7 +13,7 @@ public class AddressIndexRepository : MemorySizeCache addressIndexerDataCollection; + private readonly ILiteCollection addressIndexerDataCollection; private readonly ILogger logger; diff --git a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs index e1fbc0f55..642ac99f3 100644 --- a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs @@ -20,7 +20,6 @@ using Blockcore.Networks; using Blockcore.Utilities; using LiteDB; -using FileMode = LiteDB.FileMode; using Microsoft.Extensions.Logging; using Script = Blockcore.Consensus.ScriptInfo.Script; @@ -86,7 +85,7 @@ public class AddressIndexer : IAddressIndexer private LiteDatabase db; - private LiteCollection tipDataStore; + private ILiteCollection tipDataStore; /// A mapping between addresses and their balance changes. /// All access should be protected by . @@ -179,8 +178,8 @@ public void Initialize() string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Connection = connectionType }); this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory); diff --git a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs index 410c94845..afb25c42b 100644 --- a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs +++ b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs @@ -17,11 +17,11 @@ public sealed class AddressIndexerOutpointsRepository : MemoryCacheRepresents the output collection. /// Should be protected by - private readonly LiteCollection addressIndexerOutPointData; + private readonly ILiteCollection addressIndexerOutPointData; /// Represents the rewind data collection. /// Should be protected by - private readonly LiteCollection addressIndexerRewindData; + private readonly ILiteCollection addressIndexerRewindData; private readonly ILogger logger; @@ -119,7 +119,9 @@ public void RecordRewindData(AddressIndexerRewindData rewindData) public void PurgeOldRewindData(int height) { // Delete all in one go based on query. This is more optimal than query, iterate and delete individual records. - int purgedCount = this.addressIndexerRewindData.Delete(x => x.BlockHeight < height); + //BsonExpression predicate = Query.LT("BlockHeight", height); + // int purgedCount = this.addressIndexerRewindData.DeleteMany(predicate); + int purgedCount = this.addressIndexerRewindData.DeleteMany(x => x.BlockHeight < height); this.logger.LogInformation("Purged {0} rewind data items.", purgedCount); } diff --git a/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs b/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs index e80e5d856..38df83ce4 100644 --- a/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs +++ b/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs @@ -21,8 +21,8 @@ public class AddressIndexerOutpointsRepositoryTests public AddressIndexerOutpointsRepositoryTests() { - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".db", Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".db", Upgrade = true, Connection = connectionType }); this.repository = new AddressIndexerOutpointsRepository(db, new ExtendedLoggerFactory(), this.maxItems); } diff --git a/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerTests.cs b/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerTests.cs index 06157c97e..d8a412ac9 100644 --- a/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerTests.cs +++ b/src/Tests/Blockcore.Features.BlockStore.Tests/AddressIndexerTests.cs @@ -181,8 +181,8 @@ public void OutPointCacheCanRetrieveExisting() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); var outPoint = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 0); @@ -204,8 +204,8 @@ public void OutPointCacheCannotRetrieveNonexistent() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); Assert.False(cache.TryGetOutPointData(new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1), out OutPointData retrieved)); @@ -219,8 +219,8 @@ public void OutPointCacheEvicts() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory(), 2); Assert.Equal(0, cache.Count); @@ -270,8 +270,8 @@ public void AddressCacheCanRetrieveExisting() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); string address = "xyz"; @@ -299,8 +299,8 @@ public void AddressCacheRetrievesBlankRecordForNonexistent() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz"); @@ -318,8 +318,8 @@ public void AddressCacheEvicts() var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - LiteDB.FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? LiteDB.FileMode.Exclusive : LiteDB.FileMode.Shared; - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Mode = fileMode }); + ConnectionType connectionType = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ConnectionType.Direct : ConnectionType.Shared; + var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true, Connection = connectionType }); var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory(), 4); // Recall, each index entry counts as 1 and each balance change associated with it is an additional 1. From 16183d5c64b07f337536fce08008ce0e6677ca1e Mon Sep 17 00:00:00 2001 From: Panayiotis Savva Date: Sun, 17 Mar 2024 20:54:02 +0200 Subject: [PATCH 2/3] Query LiteDb transform query to work in v5 --- .../AddressIndexing/AddressIndexRepository.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs index 97ca4efaf..1ac5fd335 100644 --- a/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs +++ b/src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs @@ -58,7 +58,8 @@ public List GetAddressesHigherThanHeight(int height) this.SaveAllItems(); // Need to specify index name explicitly so that it gets used for the query. - IEnumerable affectedAddresses = this.addressIndexerDataCollection.Find(Query.GT("BalanceChangedHeightIndex", height)); + // Query to find documents where any BalanceChangedHeight in the BalanceChanges array is greater than 'height' + IEnumerable affectedAddresses = this.addressIndexerDataCollection.Find(Query.GT("$.BalanceChanges[*].BalanceChangedHeight ANY", height)).ToList(); // Per LiteDb documentation: // "Returning an IEnumerable your code still connected to datafile. From 153ba3731432feffc7dd3bc2febca95ac6dcca5e Mon Sep 17 00:00:00 2001 From: Panayiotis Savva Date: Sun, 17 Mar 2024 22:00:13 +0200 Subject: [PATCH 3/3] Upgrade LiteDB@5.0.15 --- src/Blockcore/Blockcore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Blockcore/Blockcore.csproj b/src/Blockcore/Blockcore.csproj index 7837e0441..8861ce493 100644 --- a/src/Blockcore/Blockcore.csproj +++ b/src/Blockcore/Blockcore.csproj @@ -24,7 +24,7 @@ - +