diff --git a/src/GWallet.Backend.Tests/ServerReference.fs b/src/GWallet.Backend.Tests/ServerReference.fs index a87110563..a44045b06 100644 --- a/src/GWallet.Backend.Tests/ServerReference.fs +++ b/src/GWallet.Backend.Tests/ServerReference.fs @@ -50,6 +50,33 @@ type ServerReference() = TimeSpan = timeSpan },dummy_now) |> Some + [] + member __.``averageBetween3DiscardingOutlier: basic test``() = + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 1m 2m 3m + Assert.That(res, Is.EqualTo 2m) + () + + [] + member __.``averageBetween3DiscardingOutlier: nuanced tests``() = + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 0m 2m 3m + Assert.That(res, Is.EqualTo 2.5m) + + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 2m 0m 3m + Assert.That(res, Is.EqualTo 2.5m) + () + + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 0m 3m 2m + Assert.That(res, Is.EqualTo 2.5m) + () + + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 3m 0m 2m + Assert.That(res, Is.EqualTo 2.5m) + () + + let res = TrustMinimizedEstimation.AverageBetween3DiscardingOutlier 3m 2m 0m + Assert.That(res, Is.EqualTo 2.5m) + () + [] member __.``order of servers is kept if non-hostname details are same``() = let serverWithHighestPriority = diff --git a/src/GWallet.Backend/Currency.fs b/src/GWallet.Backend/Currency.fs index 89b874e88..647210179 100644 --- a/src/GWallet.Backend/Currency.fs +++ b/src/GWallet.Backend/Currency.fs @@ -1,10 +1,28 @@ namespace GWallet.Backend +open System open System.Linq open System.ComponentModel open GWallet.Backend.FSharpUtil.UwpHacks +module TrustMinimizedEstimation = + let AverageBetween3DiscardingOutlier (one: decimal) (two: decimal) (three: decimal): decimal = + let sorted = List.sort [one; two; three] + let first = sorted.Item 0 + let last = sorted.Item 2 + let higher = Math.Max(first, last) + let intermediate = sorted.Item 1 + let lower = Math.Min(first, last) + + if (higher - intermediate = intermediate - lower) then + (higher + intermediate + lower) / 3m + // choose the two that are closest + elif (higher - intermediate) < (intermediate - lower) then + (higher + intermediate) / 2m + else + (lower + intermediate) / 2m + // this attribute below is for Json.NET (Newtonsoft.Json) to be able to deserialize this as a dict key [)>] type Currency = diff --git a/src/GWallet.Backend/FiatValueEstimation.fs b/src/GWallet.Backend/FiatValueEstimation.fs index 0683a30e9..1d5961014 100644 --- a/src/GWallet.Backend/FiatValueEstimation.fs +++ b/src/GWallet.Backend/FiatValueEstimation.fs @@ -1,10 +1,8 @@ namespace GWallet.Backend open System -open System.Net open FSharp.Data -open Fsdk open FSharpx.Collections open GWallet.Backend.FSharpUtil.UwpHacks @@ -1004,8 +1002,7 @@ module FiatValueEstimation = | CoinGecko | CoinDesk - let private QueryOnlineInternal currency (provider: PriceProvider): Async> = async { - use webClient = new WebClient() + let private QueryOnlineInternal currency (provider: PriceProvider): Async> = let tickerName = match currency,provider with | Currency.BTC,_ -> "bitcoin" @@ -1018,7 +1015,7 @@ module FiatValueEstimation = | Currency.DAI,PriceProvider.CoinCap -> "multi-collateral-dai" | Currency.DAI,_ -> "dai" - try + async { let baseUrl = match provider with | PriceProvider.BitPay -> @@ -1032,16 +1029,14 @@ module FiatValueEstimation = failwith "CoinDesk API only provides bitcoin price" "https://api.coindesk.com/v1/bpi/currentprice.json" let uri = Uri baseUrl - let task = webClient.DownloadStringTaskAsync uri - let! res = Async.AwaitTask task - return Some (tickerName,res) - with - | ex -> - if (FSharpUtil.FindException ex).IsSome then - return None - else - return raise <| FSharpUtil.ReRaise ex - } + + let! maybeResult = Networking.QueryRestApi uri + let tupleResult = + match maybeResult with + | None -> None + | Some result -> Some (tickerName, result) + return tupleResult + } let private QueryBitPay currency = async { diff --git a/src/GWallet.Backend/GWallet.Backend-legacy.fsproj b/src/GWallet.Backend/GWallet.Backend-legacy.fsproj index 7f6b17334..e2c466b28 100644 --- a/src/GWallet.Backend/GWallet.Backend-legacy.fsproj +++ b/src/GWallet.Backend/GWallet.Backend-legacy.fsproj @@ -69,6 +69,7 @@ + diff --git a/src/GWallet.Backend/GWallet.Backend.fsproj b/src/GWallet.Backend/GWallet.Backend.fsproj index 75e953bc2..81fefb7d7 100644 --- a/src/GWallet.Backend/GWallet.Backend.fsproj +++ b/src/GWallet.Backend/GWallet.Backend.fsproj @@ -38,6 +38,7 @@ + diff --git a/src/GWallet.Backend/Networking.fs b/src/GWallet.Backend/Networking.fs index 6317ca044..81d0948fa 100644 --- a/src/GWallet.Backend/Networking.fs +++ b/src/GWallet.Backend/Networking.fs @@ -126,6 +126,21 @@ type ServerMisconfiguredException = module Networking = + let QueryRestApi (uri: Uri) = + async { + use webClient = new WebClient() + try + let task = webClient.DownloadStringTaskAsync uri + let! result = Async.AwaitTask task + return Some result + with + | ex -> + if (FSharpUtil.FindException ex).IsSome then + return None + else + return raise <| FSharpUtil.ReRaise ex + } + let FindExceptionToRethrow (ex: Exception) (newExceptionMsg): Option = match FSharpUtil.FindException ex with | None -> diff --git a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs index fda5132fa..43b5302c6 100644 --- a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs +++ b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs @@ -122,12 +122,12 @@ module ElectrumClient = return raise <| ServerMisconfiguredException(SPrintF1 "Fee estimation returned an invalid non-positive value %M" estimateFeeResult.Result) - let amountPerKB = estimateFeeResult.Result - let satPerKB = (NBitcoin.Money (amountPerKB, NBitcoin.MoneyUnit.BTC)).ToUnit NBitcoin.MoneyUnit.Satoshi + let btcPerKB = estimateFeeResult.Result + let satPerKB = (NBitcoin.Money (btcPerKB, NBitcoin.MoneyUnit.BTC)).ToUnit NBitcoin.MoneyUnit.Satoshi let satPerB = satPerKB / (decimal 1000) Infrastructure.LogDebug <| SPrintF2 - "Electrum server gave us a fee rate of %M per KB = %M sat per B" amountPerKB satPerB - return amountPerKB + "Electrum server gave us a fee rate of %M per KB = %M sat per B" btcPerKB satPerB + return btcPerKB } let BroadcastTransaction (transactionInHex: string) (stratumServer: Async) = async { diff --git a/src/GWallet.Backend/UtxoCoin/FeeRateEstimation.fs b/src/GWallet.Backend/UtxoCoin/FeeRateEstimation.fs new file mode 100644 index 000000000..738a45100 --- /dev/null +++ b/src/GWallet.Backend/UtxoCoin/FeeRateEstimation.fs @@ -0,0 +1,183 @@ +namespace GWallet.Backend.UtxoCoin + +open System +open System.Linq + +open GWallet.Backend +open GWallet.Backend.FSharpUtil.UwpHacks + +open FSharp.Data +open NBitcoin + +module FeeRateEstimation = + + type Priority = + | Highest + | Low + + type MempoolSpaceProvider = JsonProvider<""" + { + "fastestFee": 41, + "halfHourFee": 38, + "hourFee": 35, + "economyFee": 12, + "minimumFee": 6 + } + """> + + let MempoolSpaceRestApiUri = Uri "https://mempool.space/api/v1/fees/recommended" + + type BlockchainInfoProvider = JsonProvider<""" + { + "limits": { + "min": 4, + "max": 16 + }, + "regular": 9, + "priority": 11 + } + """> + + let BlockchainInfoRestApiUri = Uri "https://api.blockchain.info/mempool/fees" + + let private ToBrandedType(feeRatePerKB: decimal) (moneyUnit: MoneyUnit): FeeRate = + try + Money(feeRatePerKB, moneyUnit) |> FeeRate + with + | ex -> + // we need more info in case this bug shows again: https://gitlab.com/nblockchain/geewallet/issues/43 + raise <| Exception(SPrintF2 "Could not create fee rate from %s %A" + (feeRatePerKB.ToString()) moneyUnit, ex) + + let private QueryFeeRateToMempoolSpace (priority: Priority): Async> = + async { + let! maybeJson = Networking.QueryRestApi MempoolSpaceRestApiUri + match maybeJson with + | None -> return None + | Some json -> + let recommendedFees = MempoolSpaceProvider.Parse json + let highPrioFeeSatsPerB = + // FIXME: at the moment of writing this, .FastestFee is even higher than what electrum servers recommend (15 vs 12) + // (and .MinimumFee and .EconomyFee (3,6) seem too low, given that mempool.space website (not API) was giving 10,11,12) + match priority with + | Highest -> recommendedFees.FastestFee + | Low -> recommendedFees.EconomyFee + |> decimal + Infrastructure.LogDebug (SPrintF1 "mempool.space API gave us a fee rate of %M sat per B" highPrioFeeSatsPerB) + let satPerKB = highPrioFeeSatsPerB * (decimal 1000) + return Some <| ToBrandedType satPerKB MoneyUnit.Satoshi + } + + let private QueryFeeRateToBlockchainInfo (priority: Priority): Async> = + async { + let! maybeJson = Networking.QueryRestApi BlockchainInfoRestApiUri + match maybeJson with + | None -> return None + | Some json -> + let recommendedFees = BlockchainInfoProvider.Parse json + let highPrioFeeSatsPerB = + // FIXME: at the moment of writing this, both priority & regular give same number wtaf -> 9 + // (and .Limits.Min was 4, which seemed too low given that mempool.space website (not API) was giving 10,11,12; + // and .Limits.Max was too high, higher than what electrum servers were suggesting: 12) + match priority with + | Highest -> recommendedFees.Priority + | Low -> recommendedFees.Regular + |> decimal + Infrastructure.LogDebug (SPrintF1 "blockchain.info API gave us a fee rate of %M sat per B" highPrioFeeSatsPerB) + let satPerKB = highPrioFeeSatsPerB * (decimal 1000) + return Some <| ToBrandedType satPerKB MoneyUnit.Satoshi + } + + let private AverageFee (feesFromDifferentServers: List): decimal = + let avg = feesFromDifferentServers.Sum() / decimal feesFromDifferentServers.Length + avg + + let private QueryFeeRateToElectrumServers (currency: Currency) (priority: Priority): Async = + async { + //querying for 1 will always return -1 surprisingly... + let numBlocksToWait = + match currency, priority with + | Currency.BTC, Low -> + 6 + | Currency.LTC, _ + | _, Highest -> + //querying for 1 will always return -1 surprisingly... + 2 + | otherCurrency, otherPrio -> + failwith <| SPrintF2 "UTXO-based currency %A not implemented ElectrumServer feeRate %A query" otherCurrency otherPrio + + let estimateFeeJob = ElectrumClient.EstimateFee numBlocksToWait + let! btcPerKiloByteForFastTrans = + Server.Query currency (QuerySettings.FeeEstimation AverageFee) estimateFeeJob None + return ToBrandedType (decimal btcPerKiloByteForFastTrans) MoneyUnit.BTC + } + + let QueryFeeRateInternal currency (priority: Priority) = + let electrumJob = + async { + try + let! result = QueryFeeRateToElectrumServers currency priority + return Some result + with + | :? NoneAvailableException -> + return None + } + + async { + match currency with + | Currency.LTC -> + let! electrumResult = electrumJob + return electrumResult + | Currency.BTC -> + let! bothJobs = Async.Parallel [electrumJob; QueryFeeRateToMempoolSpace priority; QueryFeeRateToBlockchainInfo priority] + let electrumResult = bothJobs.ElementAt 0 + let mempoolSpaceResult = bothJobs.ElementAt 1 + let blockchainInfoResult = bothJobs.ElementAt 2 + match electrumResult, mempoolSpaceResult, blockchainInfoResult with + | None, None, None -> return None + | Some feeRate, None, None -> + Infrastructure.LogDebug "Only electrum servers available for feeRate estimation" + return Some feeRate + | None, Some feeRate, None -> + Infrastructure.LogDebug "Only mempool.space API available for feeRate estimation" + return Some feeRate + | None, None, Some feeRate -> + Infrastructure.LogDebug "Only blockchain.info API available for feeRate estimation" + return Some feeRate + | None, Some restApiFeeRate1, Some restApiFeeRate2 -> + Infrastructure.LogDebug "Only REST APIs available for feeRate estimation" + let average = AverageFee [decimal restApiFeeRate1.FeePerK.Satoshi; decimal restApiFeeRate2.FeePerK.Satoshi] + let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi + Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte) + return Some averageFeeRate + | Some electrumFeeRate, Some restApiFeeRate, None -> + let average = AverageFee [decimal electrumFeeRate.FeePerK.Satoshi; decimal restApiFeeRate.FeePerK.Satoshi] + let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi + Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte) + return Some averageFeeRate + | Some electrumFeeRate, None, Some restApiFeeRate -> + let average = AverageFee [decimal electrumFeeRate.FeePerK.Satoshi; decimal restApiFeeRate.FeePerK.Satoshi] + let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi + Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte) + return Some averageFeeRate + | Some electrumFeeRate, Some restApiFeeRate1, Some restApiFeeRate2 -> + let average = + TrustMinimizedEstimation.AverageBetween3DiscardingOutlier + (decimal electrumFeeRate.FeePerK.Satoshi) + (decimal restApiFeeRate1.FeePerK.Satoshi) + (decimal restApiFeeRate2.FeePerK.Satoshi) + let averageFeeRate = ToBrandedType average MoneyUnit.Satoshi + Infrastructure.LogDebug (SPrintF1 "Average fee rate of %M sat per B" averageFeeRate.SatoshiPerByte) + return Some averageFeeRate + | currency -> + return failwith <| SPrintF1 "UTXO currency not supported yet?: %A" currency + } + + let internal EstimateFeeRate currency (priority: Priority): Async = + async { + let! maybeFeeRate = QueryFeeRateInternal currency priority + match maybeFeeRate with + | None -> return failwith "Sending when offline not supported, try sign-off?" + | Some feeRate -> return feeRate + } + diff --git a/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs b/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs index 6adb43f4c..8f0a08d64 100644 --- a/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs +++ b/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs @@ -54,6 +54,8 @@ type ArchivedUtxoAccount(currency: Currency, accountFile: FileRepresentation, module Account = + let BitcoinFeeRateDefaultPriority = FeeRateEstimation.Priority.Highest + let internal GetNetwork (currency: Currency) = if not (currency.IsUtxo()) then failwith <| SPrintF1 "Assertion failed: currency %A should be UTXO-type" currency @@ -288,7 +290,7 @@ module Account = return! EstimateFees newTxBuilder feeRate account newInputs tail } - let internal EstimateTransferFee + let private EstimateFeeForTransaction (account: IUtxoAccount) (amount: TransferAmount) (destination: string) @@ -367,23 +369,7 @@ module Account = let initiallyUsedInputs = inputs |> List.ofArray - let averageFee (feesFromDifferentServers: List): decimal = - let avg = feesFromDifferentServers.Sum() / decimal feesFromDifferentServers.Length - avg - - //querying for 1 will always return -1 surprisingly... - let estimateFeeJob = ElectrumClient.EstimateFee 2 - let! btcPerKiloByteForFastTrans = - Server.Query account.Currency (QuerySettings.FeeEstimation averageFee) estimateFeeJob None - - let feeRate = - try - Money(btcPerKiloByteForFastTrans, MoneyUnit.BTC) |> FeeRate - with - | ex -> - // we need more info in case this bug shows again: https://gitlab.com/nblockchain/geewallet/issues/43 - raise <| Exception(SPrintF1 "Could not create fee rate from %s btc per KB" - (btcPerKiloByteForFastTrans.ToString()), ex) + let! feeRate = FeeRateEstimation.EstimateFeeRate currency BitcoinFeeRateDefaultPriority let transactionBuilder = CreateTransactionAndCoinsToBeSigned account initiallyUsedInputs @@ -409,7 +395,7 @@ module Account = (destination: string) : Async = async { - let! initialFee = EstimateTransferFee account amount destination + let! initialFee = EstimateFeeForTransaction account amount destination if account.Currency <> Currency.LTC then return initialFee else