Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BlockDownloadService integration #1

Closed
wants to merge 10 commits into from
Closed

BlockDownloadService integration #1

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3a60c17
WIP: Big merge and improvements
Jan 10, 2024
908b9fa
WIP: Test
Jan 11, 2024
122272e
Merge branch 'pr/12184' into parallelDl3
Jan 23, 2024
30c3e26
Make ISourceRequest nullable in Request
Jan 23, 2024
15a7b64
Integrate BlockDownloadService
Jan 23, 2024
a27f2c9
Misc improvements and service starting
Jan 25, 2024
b8522cd
Add debug logs
Jan 25, 2024
511f7af
Set max filters in memory to 20000 for more impressive results on tes…
Jan 25, 2024
2a890d7
Merge branch 'pr/12184' into parallelDl3
Jan 26, 2024
1501268
Merge latest changes
Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions WalletWasabi.Daemon/Global.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
using WalletWasabi.WabiSabi.Client.Banning;
using WalletWasabi.WabiSabi.Client.RoundStateAwaiters;
using WalletWasabi.Wallets;
using WalletWasabi.Wallets.FilterProcessor;
using WalletWasabi.WebClients.BlockstreamInfo;
using WalletWasabi.WebClients.Wasabi;

Expand Down Expand Up @@ -111,14 +112,16 @@ public Global(string dataDir, string configFilePath, Config config)
SpecificNodeBlockProvider = new SpecificNodeBlockProvider(Network, Config.ServiceConfiguration, HttpClientFactory.TorEndpoint);
P2PNodesManager = new P2PNodesManager(Network, HostedServices.Get<P2pNetwork>().Nodes, HttpClientFactory.IsTorEnabled);

var blockProvider = new SmartBlockProvider(
BitcoinStore.BlockRepository,
BitcoinCoreNode?.RpcClient is null ? null : new RpcBlockProvider(BitcoinCoreNode.RpcClient),
SpecificNodeBlockProvider,
new P2PBlockProvider(P2PNodesManager),
Cache);
var trustedFullNodeBlockProviders = BitcoinCoreNode?.RpcClient is null
? new IBlockProvider[] { SpecificNodeBlockProvider }
: new IBlockProvider[] { new RpcBlockProvider(BitcoinCoreNode.RpcClient), SpecificNodeBlockProvider };

WalletManager = new WalletManager(config.Network, DataDir, new WalletDirectories(Config.Network, DataDir), BitcoinStore, Synchronizer, HostedServices.Get<HybridFeeProvider>(), blockProvider, config.ServiceConfiguration);
BlockDownloadService = new BlockDownloadService(
fileSystemBlockRepository: BitcoinStore.BlockRepository,
trustedFullNodeBlockProviders: trustedFullNodeBlockProviders,
new P2PBlockProvider(P2PNodesManager));

WalletManager = new WalletManager(config.Network, DataDir, new WalletDirectories(Config.Network, DataDir), BitcoinStore, Synchronizer, HostedServices.Get<HybridFeeProvider>(), BlockDownloadService, config.ServiceConfiguration);
TransactionBroadcaster = new TransactionBroadcaster(Network, BitcoinStore, HttpClientFactory, WalletManager);

CoinPrison = CoinPrison.CreateOrLoadFromFile(DataDir);
Expand Down Expand Up @@ -151,6 +154,7 @@ public Global(string dataDir, string configFilePath, Config config)
public TransactionBroadcaster TransactionBroadcaster { get; set; }
public CoinJoinProcessor? CoinJoinProcessor { get; set; }
private SpecificNodeBlockProvider SpecificNodeBlockProvider { get; }
private BlockDownloadService BlockDownloadService { get; }
private P2PNodesManager P2PNodesManager { get; }
private TorProcessManager? TorManager { get; set; }
public CoreNode? BitcoinCoreNode { get; private set; }
Expand Down Expand Up @@ -436,6 +440,12 @@ public async Task DisposeAsync()
Logger.LogInfo($"{nameof(SpecificNodeBlockProvider)} is disposed.");
}

if (BlockDownloadService is { } blockDownloadService)
{
blockDownloadService.Dispose();
Logger.LogInfo($"{nameof(BlockDownloadService)} is disposed.");
}

if (CoinJoinProcessor is { } coinJoinProcessor)
{
coinJoinProcessor.Dispose();
Expand Down
20 changes: 10 additions & 10 deletions WalletWasabi.Tests/IntegrationTests/P2pTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
using WalletWasabi.Stores;
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Wallets;
using WalletWasabi.Wallets.BlockProvider;
using WalletWasabi.Wallets.FilterProcessor;
using WalletWasabi.WebClients.Wasabi;
using Xunit;

Expand Down Expand Up @@ -112,12 +114,10 @@ public async Task TestServicesAsync(string networkString)
IFileSystemBlockRepository blockRepository = bitcoinStore.BlockRepository;
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfig, httpClientFactory.TorEndpoint);

IBlockProvider blockProvider = new SmartBlockProvider(
blockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

using Wallet wallet = Wallet.CreateAndRegisterServices(
network,
Expand All @@ -127,7 +127,7 @@ public async Task TestServicesAsync(string networkString)
dataDir,
new ServiceConfiguration(new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold)),
feeProvider,
blockProvider);
blockDownloadService);
Assert.True(Directory.Exists(blocks.BlocksFolderPath));

try
Expand All @@ -144,18 +144,18 @@ public async Task TestServicesAsync(string networkString)

nodes.Connect();

var downloadTasks = new List<Task<Block>>();
var downloadTasks = new List<Task<BlockDownloadService.IResult>>();
using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(4));
foreach (var hash in blocksToDownload)
{
downloadTasks.Add(blockProvider.GetBlockAsync(hash, cts.Token));
downloadTasks.Add(blockDownloadService.TryGetBlockAsync(P2pSourceRequest.Automatic, hash, new Priority(SyncType.Complete), uint.MaxValue, cts.Token));
}

await nodeConnectionAwaiter.WaitAsync(TimeSpan.FromMinutes(3));

var i = 0;
var hashArray = blocksToDownload.ToArray();
foreach (var block in await Task.WhenAll(downloadTasks))
foreach (var result in await Task.WhenAll(downloadTasks))
{
Assert.True(File.Exists(Path.Combine(blocks.BlocksFolderPath, hashArray[i].ToString())));
i++;
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/BuildTransactionReorgsTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using WalletWasabi.Stores;
using WalletWasabi.Tests.XunitConfiguration;
using WalletWasabi.Wallets;
using WalletWasabi.Wallets.FilterProcessor;
using WalletWasabi.WebClients.Wasabi;
using Xunit;

Expand Down Expand Up @@ -75,14 +76,12 @@ public async Task BuildTransactionReorgsTestAsync()
using MemoryCache cache = CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

var baseTip = await rpc.GetBestBlockHashAsync();
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/BuildTransactionValidationsTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using WalletWasabi.Stores;
using WalletWasabi.Tests.XunitConfiguration;
using WalletWasabi.Wallets;
using WalletWasabi.Wallets.FilterProcessor;
using WalletWasabi.WebClients.Wasabi;
using Xunit;

Expand Down Expand Up @@ -74,14 +75,12 @@ public async Task BuildTransactionValidationsTestAsync()
using MemoryCache cache = CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
p2PBlockProvider: new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockProvider);
using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockDownloadService);
wallet.NewFiltersProcessed += setup.Wallet_NewFiltersProcessed;

using Key key = new();
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/CancelTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using WalletWasabi.Helpers;
using WalletWasabi.Logging;
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -73,14 +74,12 @@ public async Task CancelTestsAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/MaxFeeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Tests.XunitConfiguration;
using WalletWasabi.Wallets;
using WalletWasabi.Wallets.FilterProcessor;
using WalletWasabi.WebClients.Wasabi;
using Xunit;

Expand Down Expand Up @@ -73,14 +74,12 @@ public async Task CalculateMaxFeeTestAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/ReceiveSpeedupTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
using WalletWasabi.Helpers;
using WalletWasabi.Blockchain.Transactions;
using WalletWasabi.Exceptions;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -75,14 +76,12 @@ public async Task ReceiveSpeedupTestsAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

try
Expand Down
12 changes: 5 additions & 7 deletions WalletWasabi.Tests/RegressionTests/ReplaceByFeeTxTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using WalletWasabi.WebClients.Wasabi;
using Xunit;
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -68,17 +69,14 @@ public async Task ReplaceByFeeTxTestAsync()
// 5. Create wallet service.
var workDir = Common.GetWorkDir();

using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockProvider);
using var wallet = Wallet.CreateAndRegisterServices(network, bitcoinStore, keyManager, synchronizer, workDir, serviceConfiguration, feeProvider, blockDownloadService);
wallet.NewFiltersProcessed += setup.Wallet_NewFiltersProcessed;

Assert.Empty(wallet.Coins);
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/SelfSpendSpeedupTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
using WalletWasabi.Helpers;
using WalletWasabi.Blockchain.Transactions;
using WalletWasabi.Exceptions;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -75,14 +76,12 @@ public async Task SelfSpendSpeedupTestsAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/SendSpeedupTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using WalletWasabi.Blockchain.Transactions;
using WalletWasabi.Exceptions;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -76,14 +77,12 @@ public async Task SendSpeedupTestsAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/SendTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using WalletWasabi.WebClients.Wasabi;
using Xunit;
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -73,14 +74,12 @@ public async Task SendTestsAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
11 changes: 5 additions & 6 deletions WalletWasabi.Tests/RegressionTests/SpendUnconfirmedTxTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using WalletWasabi.WebClients.Wasabi;
using Xunit;
using WalletWasabi.Tests.Helpers;
using WalletWasabi.Wallets.FilterProcessor;

namespace WalletWasabi.Tests.RegressionTests;

Expand Down Expand Up @@ -73,14 +74,12 @@ public async Task SpendUnconfirmedTxTestAsync()
using MemoryCache cache = BitcoinFactory.CreateMemoryCache();
await using SpecificNodeBlockProvider specificNodeBlockProvider = new(network, serviceConfiguration, httpClientFactory.TorEndpoint);

var blockProvider = new SmartBlockProvider(
var blockDownloadService = new BlockDownloadService(
bitcoinStore.BlockRepository,
rpcBlockProvider: null,
specificNodeBlockProvider,
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled),
cache);
new IBlockProvider[] { specificNodeBlockProvider },
new P2PBlockProvider(network, nodes, httpClientFactory.IsTorEnabled));

WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockProvider, serviceConfiguration);
WalletManager walletManager = new(network, workDir, new WalletDirectories(network, workDir), bitcoinStore, synchronizer, feeProvider, blockDownloadService, serviceConfiguration);
walletManager.Initialize();

// Get some money, make it confirm.
Expand Down
Loading