WIP: Big merge and improvements

WalletWasabi.Daemon/Global.cs

@@ -32,6 +32,7 @@
 using WalletWasabi.WabiSabi.Client.Banning;
 using WalletWasabi.WabiSabi.Client.RoundStateAwaiters;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.BlockstreamInfo;
 using WalletWasabi.WebClients.Wasabi;
 
@@ -110,7 +111,9 @@ public Global(string dataDir, string configFilePath, Config config)
 			new P2PBlockProvider(Network, HostedServices.Get<P2pNetwork>().Nodes, HttpClientFactory.IsTorEnabled),
 			Cache);
 
-		WalletManager = new WalletManager(config.Network, DataDir, new WalletDirectories(Config.Network, DataDir), BitcoinStore, Synchronizer, HostedServices.Get<HybridFeeProvider>(), blockProvider, config.ServiceConfiguration);
+		BlockDownloadService = new BlockDownloadService(blockProvider);
+
+		WalletManager = new WalletManager(config.Network, DataDir, new WalletDirectories(Config.Network, DataDir), BitcoinStore, Synchronizer, HostedServices.Get<HybridFeeProvider>(), BlockDownloadService, config.ServiceConfiguration);
 		TransactionBroadcaster = new TransactionBroadcaster(Network, BitcoinStore, HttpClientFactory, WalletManager);
 
 		CoinPrison = CoinPrison.CreateOrLoadFromFile(DataDir);
@@ -143,6 +146,7 @@ public Global(string dataDir, string configFilePath, Config config)
 	public TransactionBroadcaster TransactionBroadcaster { get; set; }
 	public CoinJoinProcessor? CoinJoinProcessor { get; set; }
 	private SpecificNodeBlockProvider SpecificNodeBlockProvider { get; }
+	private BlockDownloadService BlockDownloadService { get; }
 	private TorProcessManager? TorManager { get; set; }
 	public CoreNode? BitcoinCoreNode { get; private set; }
 	public TorStatusChecker TorStatusChecker { get; set; }
@@ -427,6 +431,12 @@ public async Task DisposeAsync()
 					Logger.LogInfo($"{nameof(SpecificNodeBlockProvider)} is disposed.");
 				}
 
+				if (BlockDownloadService is { } blockDownloadService)
+				{
+					blockDownloadService.Dispose();
+					Logger.LogInfo($"{nameof(BlockDownloadService)} is disposed.");
+				}
+
 				if (CoinJoinProcessor is { } coinJoinProcessor)
 				{
 					coinJoinProcessor.Dispose();

WalletWasabi.Tests/IntegrationTests/P2pTests.cs

@@ -20,6 +20,7 @@
 using WalletWasabi.Stores;
 using WalletWasabi.Tests.Helpers;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 
@@ -127,7 +128,7 @@ public async Task TestServicesAsync(string networkString)
 			dataDir,
 			new ServiceConfiguration(new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold)),
 			feeProvider,
-			blockProvider);
+			new BlockDownloadService(blockProvider));
 		Assert.True(Directory.Exists(blocks.BlocksFolderPath));
 
 		try

WalletWasabi.Tests/RegressionTests/BuildTransactionReorgsTest.cs

@@ -21,6 +21,7 @@
 using WalletWasabi.Stores;
 using WalletWasabi.Tests.XunitConfiguration;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 
@@ -82,7 +83,7 @@ public async Task BuildTransactionReorgsTestAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		var baseTip = await rpc.GetBestBlockHashAsync();

WalletWasabi.Tests/RegressionTests/BuildTransactionValidationsTest.cs

@@ -21,6 +21,7 @@
 using WalletWasabi.Stores;
 using WalletWasabi.Tests.XunitConfiguration;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 
@@ -81,7 +82,7 @@ public async Task BuildTransactionValidationsTestAsync()
 			p2PBlockProvider: new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockProvider);
+		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, new BlockDownloadService(blockProvider));
 		wallet.NewFiltersProcessed += setup.Wallet_NewFiltersProcessed;
 
 		using Key key = new();

WalletWasabi.Tests/RegressionTests/CancelTests.cs

@@ -22,6 +22,7 @@
 using WalletWasabi.Helpers;
 using WalletWasabi.Logging;
 using WalletWasabi.Tests.Helpers;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -80,7 +81,7 @@ public async Task CancelTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/MaxFeeTests.cs

@@ -20,6 +20,7 @@
 using WalletWasabi.Tests.Helpers;
 using WalletWasabi.Tests.XunitConfiguration;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 
@@ -80,7 +81,7 @@ public async Task CalculateMaxFeeTestAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/ReceiveSpeedupTests.cs

@@ -24,6 +24,7 @@
 using WalletWasabi.Helpers;
 using WalletWasabi.Blockchain.Transactions;
 using WalletWasabi.Exceptions;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -82,7 +83,7 @@ public async Task ReceiveSpeedupTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		try

WalletWasabi.Tests/RegressionTests/ReplaceByFeeTxTest.cs

@@ -22,6 +22,7 @@
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 using WalletWasabi.Tests.Helpers;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -78,7 +79,7 @@ public async Task ReplaceByFeeTxTestAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockProvider);
+		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, new BlockDownloadService(blockProvider));
 		wallet.NewFiltersProcessed += setup.Wallet_NewFiltersProcessed;
 
 		Assert.Empty(wallet.Coins);

WalletWasabi.Tests/RegressionTests/SelfSpendSpeedupTests.cs

@@ -24,6 +24,7 @@
 using WalletWasabi.Helpers;
 using WalletWasabi.Blockchain.Transactions;
 using WalletWasabi.Exceptions;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -82,7 +83,7 @@ public async Task SelfSpendSpeedupTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/SendSpeedupTests.cs

@@ -25,6 +25,7 @@
 using WalletWasabi.Blockchain.Transactions;
 using WalletWasabi.Exceptions;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -83,7 +84,7 @@ public async Task SendSpeedupTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/SendTests.cs

@@ -22,6 +22,7 @@
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 using WalletWasabi.Tests.Helpers;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -80,7 +81,7 @@ public async Task SendTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/SpendUnconfirmedTxTests.cs

@@ -22,6 +22,7 @@
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 using WalletWasabi.Tests.Helpers;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.RegressionTests;
 
@@ -80,7 +81,7 @@ public async Task SpendUnconfirmedTxTestAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
+		WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, new BlockDownloadService(blockProvider), serviceConfiguration);
 		walletManager.Initialize();
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/RegressionTests/WalletTests.cs

@@ -17,6 +17,7 @@
 using WalletWasabi.Stores;
 using WalletWasabi.Tests.XunitConfiguration;
 using WalletWasabi.Wallets;
+using WalletWasabi.Wallets.FilterProcessor;
 using WalletWasabi.WebClients.Wasabi;
 using Xunit;
 
@@ -80,7 +81,7 @@ public async Task WalletTestsAsync()
 			new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
 			cache);
 
-		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, setup.ServiceConfiguration, feeProvider, blockProvider);
+		using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, setup.ServiceConfiguration, feeProvider, new BlockDownloadService(blockProvider));
 		wallet.NewFiltersProcessed += setup.Wallet_NewFiltersProcessed;
 
 		// Get some money, make it confirm.

WalletWasabi.Tests/UnitTests/Wallet/FilterProcessor/ParallelBlockDownloadServiceTests.cs → WalletWasabi.Tests/UnitTests/Wallet/FilterProcessor/BlockDownloadServiceTests.cs

@@ -10,9 +10,9 @@
 namespace WalletWasabi.Tests.UnitTests.Wallet.FilterProcessor;
 
 /// <summary>
-/// Tests for <see cref="ParallelBlockDownloadService"/>.
+/// Tests for <see cref="BlockDownloadService"/>.
 /// </summary>
-public class ParallelBlockDownloadServiceTests
+public class BlockDownloadServiceTests
 {
 	/// <summary>
 	/// Verifies that blocks are being downloaded in parallel. Moreover, we attempt to download a block again if it fails to download.
@@ -39,7 +39,7 @@ public async Task BlockDownloadingTestAsync()
 		Mock<IBlockProvider> mockBlockProvider = new(MockBehavior.Strict);
 		IBlockProvider blockProvider = mockBlockProvider.Object;
 
-		using (ParallelBlockDownloadService service = new(blockProvider, maximumParallelTasks: 3))
+		using (BlockDownloadService service = new(blockProvider, maximumParallelTasks: 3))
 		{
 			// Handling of downloading of block1.
 			_ = mockBlockProvider.Setup(c => c.TryGetBlockAsync(blockHash1, It.IsAny<CancellationToken>()))
@@ -71,10 +71,10 @@ public async Task BlockDownloadingTestAsync()
 				.ReturnsAsync(block4);
 
 			await service.StartAsync(testCts.Token);
-			service.Enqueue(blockHash1, blockHeight: 610_001);
-			service.Enqueue(blockHash2, blockHeight: 610_002);
-			service.Enqueue(blockHash3, blockHeight: 610_003);
-			service.Enqueue(blockHash4, blockHeight: 610_004);
+			service.Enqueue(blockHash1, blockHeight: 610_001, SyncType.Turbo);
+			service.Enqueue(blockHash2, blockHeight: 610_002, SyncType.Turbo);
+			service.Enqueue(blockHash3, blockHeight: 610_003, SyncType.Turbo);
+			service.Enqueue(blockHash4, blockHeight: 610_004, SyncType.Turbo);
 
 			await block2FirstRequestTcs.Task.WaitAsync(testCts.Token);
 			block2DelayTcs.SetResult();
@@ -90,67 +90,6 @@ public async Task BlockDownloadingTestAsync()
 		mockBlockProvider.VerifyAll();
 	}
 
-	/// <summary>
-	/// Verifies that a block is attempted to be downloaded at most <see cref="ParallelBlockDownloadService.MaxFailedAttempts"/> times.
-	/// </summary>
-	[Fact]
-	public async Task MaxBlockDownloadAttemptsAsync()
-	{
-		using CancellationTokenSource testCts = new(TimeSpan.FromSeconds(400));
-
-		uint256 blockHash1 = uint256.One;
-		Block block1 = Network.Main.Consensus.ConsensusFactory.CreateBlock();
-
-		TaskCompletionSource block1LastFailedAttemptTcs = new();
-
-		Mock<IBlockProvider> mockBlockProvider = new(MockBehavior.Strict);
-		IBlockProvider blockProvider = mockBlockProvider.Object;
-
-		int actualAttempts = 0;
-		bool testFailed = false;
-
-		using (ParallelBlockDownloadService service = new(blockProvider, maximumParallelTasks: 3))
-		{
-			// Handling of downloading of block1.
-			_ = mockBlockProvider.Setup(c => c.TryGetBlockAsync(blockHash1, It.IsAny<CancellationToken>()))
-				.ReturnsAsync((uint256 blockHash, CancellationToken cancellationToken) =>
-				{
-					actualAttempts++;
-
-					switch (actualAttempts)
-					{
-						case < ParallelBlockDownloadService.MaxFailedAttempts:
-							break;
-						case ParallelBlockDownloadService.MaxFailedAttempts:
-							block1LastFailedAttemptTcs.SetResult();
-							break;
-						case > ParallelBlockDownloadService.MaxFailedAttempts:
-							testFailed = true; // This should never happen.
-							break;
-					}
-
-					return null;
-				});
-
-			await service.StartAsync(testCts.Token);
-
-			service.Enqueue(blockHash1, blockHeight: 610_001);
-
-			// Wait for all failed attempts.
-			await block1LastFailedAttemptTcs.Task.WaitAsync(testCts.Token);
-
-			await service.StopAsync(testCts.Token);
-			await service.ExecuteTask!.WaitAsync(testCts.Token);
-
-			// Make sure that the block is really dropped.
-			Assert.Equal(0, service.BlocksToDownload.Count);
-		}
-
-		Assert.False(testFailed);
-
-		mockBlockProvider.VerifyAll();
-	}
-
 	[Fact]
 	public void RemoveBlocks()
 	{
@@ -167,18 +106,18 @@ public void RemoveBlocks()
 		Mock<IBlockProvider> mockBlockProvider = new(MockBehavior.Strict);
 		IBlockProvider blockProvider = mockBlockProvider.Object;
 
-		using ParallelBlockDownloadService service = new(blockProvider, maximumParallelTasks: 3);
+		using BlockDownloadService service = new(blockProvider, maximumParallelTasks: 3);
 
 		// Intentionally, tested before the service is started just to smoke test that the queue is modified.
-		service.Enqueue(blockHash1, blockHeight: 610_001);
-		service.Enqueue(blockHash2, blockHeight: 610_002);
-		service.Enqueue(blockHash3, blockHeight: 610_003);
-		service.Enqueue(blockHash4, blockHeight: 610_004);
+		service.Enqueue(blockHash1, blockHeight: 610_001, SyncType.Turbo);
+		service.Enqueue(blockHash2, blockHeight: 610_002, SyncType.Turbo);
+		service.Enqueue(blockHash3, blockHeight: 610_003, SyncType.Turbo);
+		service.Enqueue(blockHash4, blockHeight: 610_004, SyncType.Turbo);
 
 		// Remove blocks >= 610_003.
 		service.RemoveBlocks(maxBlockHeight: 610_003);
 
-		ParallelBlockDownloadService.Request[] actualRequests = service.BlocksToDownload.UnorderedItems
+		BlockDownloadService.Request[] actualRequests = service.BlocksToDownload.UnorderedItems
 			.Select(x => x.Element)
 			.OrderBy(x => x.BlockHeight)
 			.ToArray();

WalletWasabi.Tests/UnitTests/Wallet/WalletBuilder.cs

@@ -19,6 +19,7 @@
 using WalletWasabi.Tests.Helpers;
 using System.IO;
 using System.Linq;
+using WalletWasabi.Wallets.FilterProcessor;
 
 namespace WalletWasabi.Tests.UnitTests.Wallet;
 
@@ -60,8 +61,9 @@ public WalletBuilder(MockNode node, [CallerMemberName] string callerName = "NN")
 		WasabiSynchronizer synchronizer = new(requestInterval: TimeSpan.FromSeconds(3), 1000, BitcoinStore, HttpClientFactory);
 		HybridFeeProvider feeProvider = new(synchronizer, null);
 		SmartBlockProvider blockProvider = new(BitcoinStore.BlockRepository, rpcBlockProvider: null, null, null, Cache);
+		BlockDownloadService blockDownloadService = new BlockDownloadService(blockProvider);
 
-		return WalletWasabi.Wallets.Wallet.CreateAndRegisterServices(Network.RegTest, BitcoinStore, keyManager, synchronizer, DataDir, serviceConfiguration, feeProvider, blockProvider);
+		return WalletWasabi.Wallets.Wallet.CreateAndRegisterServices(Network.RegTest, BitcoinStore, keyManager, synchronizer, DataDir, serviceConfiguration, feeProvider, blockDownloadService);
 	}
 
 	public async ValueTask DisposeAsync()