Remove network stats computation (#2094)

We introduced a task to regularly compute network statistics (mostly about
channel parameters such as expiry and fees).

The goal was to use this information in the MPP split algorithm to decide
whether to split a payment or not.

But we haven't used it, and I'm not sure anymore that it's useful at all.

If node operators are interested in network statistics, an ad-hoc
on-the-fly computation would make more sense.

docs/release-notes/eclair-vnext.md

@@ -29,10 +29,12 @@ Messages sent to Eclair can be read with the websocket API.
 #### Timestamps
 
 All timestamps are now returned as an object with two attributes:
+
 - `iso`: ISO-8601 format with GMT time zone. Precision may be second or millisecond depending on the timestamp.
 - `unix`: seconds since epoch formats (seconds since epoch). Precision is always second.
 
 Examples:
+
 - second-precision timestamp:
   - before:
   ```json
@@ -73,6 +75,14 @@ It expects `--route` a list of `nodeId` to send the message through, the last on
 It also accepts `--pathId` as an encoded TLV stream in hexadecimal.
 Sending to a blinded route (as a reply to a previous message) is not supported.
 
+#### Balance
+
+The detailed balance json format has been slightly updated for channels in state `normal` and `shutdown`, and `closing`.
+
+Amounts corresponding to incoming htlcs for which we knew the preimage were previously included in `toLocal`, they are
+now grouped with outgoing htlcs amounts and the field has been renamed from `htlcOut` to `htlcs`.
+
+#### Miscellaneous
 
 This release contains many other API updates:
 
@@ -83,16 +93,10 @@ This release contains many other API updates:
 - `findroute`, `findroutetonode` and `findroutebetweennodes` now accept `--maxFeeMsat` to specify an upper bound of fees (#1969)
 - `getsentinfo` output includes `failedNode` field for all failed routes
 - for `payinvoice` and `sendtonode`, `--feeThresholdSat` has been renamed to `--maxFeeFlatSat`
+- the `networkstats` API has been removed
 
 Have a look at our [API documentation](https://acinq.github.io/eclair) for more details.
 
-#### Balance
-
-The detailed balance json format has been slightly updated for channels in state `normal` and `shutdown`, and `closing`.
-
-Amounts corresponding to incoming htlcs for which we knew the preimage were previously included in `toLocal`, they are
-now grouped with outgoing htlcs amounts and the field has been renamed from `htlcOut` to `htlcs`.
-
 ### Miscellaneous improvements and bug fixes
 
 - Eclair now supports cookie authentication for Bitcoin Core RPC (#1986)

eclair-core/eclair-cli

@@ -53,7 +53,6 @@ and COMMAND is one of the available commands:
     - findroute
     - findroutetonode
     - findroutebetweennodes
-    - networkstats
     - nodes
 
   === Invoice ===

eclair-core/src/main/resources/reference.conf

@@ -214,7 +214,6 @@ eclair {
   router {
     channel-exclude-duration = 60 seconds // when a temporary channel failure is returned, we exclude the channel from our payment routes for this duration
     broadcast-interval = 60 seconds // see BOLT #7
-    network-stats-interval = 6 hours // frequency at which we refresh global network statistics (expensive operation)
     init-timeout = 5 minutes
 
     sync {

eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala

@@ -42,8 +42,8 @@ import fr.acinq.eclair.payment.receive.MultiPartHandler.ReceivePayment
 import fr.acinq.eclair.payment.relay.Relayer.{GetOutgoingChannels, OutgoingChannels, RelayFees, UsableBalance}
 import fr.acinq.eclair.payment.send.MultiPartPaymentLifecycle.PreimageReceived
 import fr.acinq.eclair.payment.send.PaymentInitiator.{SendPaymentToNode, SendPaymentToRoute, SendPaymentToRouteResponse, SendSpontaneousPayment}
+import fr.acinq.eclair.router.Router
 import fr.acinq.eclair.router.Router._
-import fr.acinq.eclair.router.{NetworkStats, Router}
 import fr.acinq.eclair.wire.protocol._
 import grizzled.slf4j.Logging
 import scodec.bits.ByteVector
@@ -126,8 +126,6 @@ trait Eclair {
 
   def channelStats(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[Stats]]
 
-  def networkStats()(implicit timeout: Timeout): Future[Option[NetworkStats]]
-
   def getInvoice(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[PaymentRequest]]
 
   def pendingInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[PaymentRequest]]
@@ -395,8 +393,6 @@ class EclairImpl(appKit: Kit) extends Eclair with Logging {
     Future(appKit.nodeParams.db.audit.stats(from.toTimestampMilli, to.toTimestampMilli))
   }
 
-  override def networkStats()(implicit timeout: Timeout): Future[Option[NetworkStats]] = (appKit.router ? GetNetworkStats).mapTo[GetNetworkStatsResponse].map(_.stats)
-
   override def allInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[PaymentRequest]] = Future {
     appKit.nodeParams.db.payments.listIncomingPayments(from.toTimestampMilli, to.toTimestampMilli).map(_.paymentRequest)
   }
@@ -413,8 +409,6 @@ class EclairImpl(appKit: Kit) extends Eclair with Logging {
     Future.fromTry(appKit.nodeParams.db.payments.removeIncomingPayment(paymentHash).map(_ => s"deleted invoice $paymentHash"))
   }
 
-
-
   /**
    * Send a request to a channel and expect a response.
    *
@@ -522,9 +516,9 @@ class EclairImpl(appKit: Kit) extends Eclair with Logging {
             Nil,
             userCustomTlvs)
         (appKit.switchboard ? OnionMessages.SendMessage(nextNodeId, message)).mapTo[MessageRelay.Status].map {
-            case MessageRelay.Success => SendOnionMessageResponse(sent = true, None)
-            case MessageRelay.Failure(f) => SendOnionMessageResponse(sent = false, Some(f.toString))
-          }
+          case MessageRelay.Success => SendOnionMessageResponse(sent = true, None)
+          case MessageRelay.Failure(f) => SendOnionMessageResponse(sent = false, Some(f.toString))
+        }
       case Attempt.Failure(cause) => Future.successful(SendOnionMessageResponse(sent = false, Some(s"the `content` field is invalid, it must contain encoded tlvs: ${cause.message}")))
     }
 

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -455,7 +455,6 @@ object NodeParams extends Logging {
       routerConf = RouterConf(
         channelExcludeDuration = FiniteDuration(config.getDuration("router.channel-exclude-duration").getSeconds, TimeUnit.SECONDS),
         routerBroadcastInterval = FiniteDuration(config.getDuration("router.broadcast-interval").getSeconds, TimeUnit.SECONDS),
-        networkStatsRefreshInterval = FiniteDuration(config.getDuration("router.network-stats-interval").getSeconds, TimeUnit.SECONDS),
         requestNodeAnnouncements = config.getBoolean("router.sync.request-node-announcements"),
         encodingType = routerSyncEncodingType,
         channelRangeChunkSize = config.getInt("router.sync.channel-range-chunk-size"),

eclair-core/src/main/scala/fr/acinq/eclair/remote/EclairInternalsSerializer.scala

@@ -83,13 +83,12 @@ object EclairInternalsSerializer {
       ("experimentPercentage" | int32)).as[PathFindingConf]
 
   val pathFindingExperimentConfCodec: Codec[PathFindingExperimentConf] = (
-    ("experiments" | listOfN(int32, pathFindingConfCodec).xmap[Map[String, PathFindingConf]](_.map(e => (e.experimentName -> e)).toMap, _.values.toList))
+    "experiments" | listOfN(int32, pathFindingConfCodec).xmap[Map[String, PathFindingConf]](_.map(e => e.experimentName -> e).toMap, _.values.toList)
     ).as[PathFindingExperimentConf]
 
   val routerConfCodec: Codec[RouterConf] = (
     ("channelExcludeDuration" | finiteDurationCodec) ::
       ("routerBroadcastInterval" | finiteDurationCodec) ::
-      ("networkStatsRefreshInterval" | finiteDurationCodec) ::
       ("requestNodeAnnouncements" | bool(8)) ::
       ("encodingType" | discriminated[EncodingType].by(uint8)
         .typecase(0, provide(EncodingType.UNCOMPRESSED))

eclair-core/src/main/scala/fr/acinq/eclair/router/Router.scala

@@ -35,7 +35,7 @@ import fr.acinq.eclair.payment.PaymentRequest.ExtraHop
 import fr.acinq.eclair.remote.EclairInternalsSerializer.RemoteTypes
 import fr.acinq.eclair.router.Graph.GraphStructure.DirectedGraph
 import fr.acinq.eclair.router.Graph.{HeuristicsConstants, WeightRatios}
-import fr.acinq.eclair.router.Monitoring.{Metrics, Tags}
+import fr.acinq.eclair.router.Monitoring.Metrics
 import fr.acinq.eclair.wire.protocol._
 import kamon.context.Context
 
@@ -63,7 +63,6 @@ class Router(val nodeParams: NodeParams, watcher: typed.ActorRef[ZmqWatcher.Comm
 
   startTimerWithFixedDelay(TickBroadcast.toString, TickBroadcast, nodeParams.routerConf.routerBroadcastInterval)
   startTimerWithFixedDelay(TickPruneStaleChannels.toString, TickPruneStaleChannels, 1 hour)
-  startTimerWithFixedDelay(TickComputeNetworkStats.toString, TickComputeNetworkStats, nodeParams.routerConf.networkStatsRefreshInterval)
 
   val db: NetworkDb = nodeParams.db.network
 
@@ -99,21 +98,17 @@ class Router(val nodeParams: NodeParams, watcher: typed.ActorRef[ZmqWatcher.Comm
     val nodeAnn = Announcements.makeNodeAnnouncement(nodeParams.privateKey, nodeParams.alias, nodeParams.color, nodeParams.publicAddresses, nodeParams.features)
     self ! nodeAnn
 
-    log.info(s"computing network stats...")
-    val stats = NetworkStats.computeStats(initChannels.values)
-
     log.info(s"initialization completed, ready to process messages")
     Try(initialized.map(_.success(Done)))
-    startWith(NORMAL, Data(initNodes, initChannels, stats, Stash(Map.empty, Map.empty), rebroadcast = Rebroadcast(channels = Map.empty, updates = Map.empty, nodes = Map.empty), awaiting = Map.empty, privateChannels = Map.empty, excludedChannels = Set.empty, graph, sync = Map.empty))
+    startWith(NORMAL, Data(initNodes, initChannels, Stash(Map.empty, Map.empty), rebroadcast = Rebroadcast(channels = Map.empty, updates = Map.empty, nodes = Map.empty), awaiting = Map.empty, privateChannels = Map.empty, excludedChannels = Set.empty, graph, sync = Map.empty))
   }
 
   when(NORMAL) {
 
     case Event(SyncProgress(progress), d: Data) =>
       Metrics.SyncProgress.withoutTags().update(100 * progress)
-      if (d.stats.isEmpty && progress == 1.0 && d.channels.nonEmpty) {
-        log.info("initial routing sync done: computing network statistics")
-        self ! TickComputeNetworkStats
+      if (progress == 1.0 && d.channels.nonEmpty) {
+        log.info("initial routing sync done")
       }
       stay()
 
@@ -122,10 +117,6 @@ class Router(val nodeParams: NodeParams, watcher: typed.ActorRef[ZmqWatcher.Comm
       sender() ! RoutingState(d.channels.values, d.nodes.values)
       stay()
 
-    case Event(GetNetworkStats, d: Data) =>
-      sender() ! GetNetworkStatsResponse(d.stats)
-      stay()
-
     case Event(GetRoutingStateStreaming, d) =>
       val listener = sender()
       d.nodes
@@ -164,18 +155,6 @@ class Router(val nodeParams: NodeParams, watcher: typed.ActorRef[ZmqWatcher.Comm
         stay() using d.copy(rebroadcast = Rebroadcast(channels = Map.empty, updates = Map.empty, nodes = Map.empty))
       }
 
-    case Event(TickComputeNetworkStats, d) =>
-      if (d.channels.nonEmpty) {
-        Metrics.Nodes.withoutTags().update(d.nodes.size)
-        Metrics.Channels.withTag(Tags.Announced, value = true).update(d.channels.size)
-        Metrics.Channels.withTag(Tags.Announced, value = false).update(d.privateChannels.size)
-        log.info("re-computing network statistics")
-        stay() using d.copy(stats = NetworkStats.computeStats(d.channels.values))
-      } else {
-        log.debug("cannot compute network statistics: no public channels available")
-        stay()
-      }
-
     case Event(TickPruneStaleChannels, d) =>
       stay() using StaleChannels.handlePruneStaleChannels(d, nodeParams.db.network, nodeParams.currentBlockHeight)
 
@@ -320,7 +299,6 @@ object Router {
 
   case class RouterConf(channelExcludeDuration: FiniteDuration,
                         routerBroadcastInterval: FiniteDuration,
-                        networkStatsRefreshInterval: FiniteDuration,
                         requestNodeAnnouncements: Boolean,
                         encodingType: EncodingType,
                         channelRangeChunkSize: Int,
@@ -535,8 +513,6 @@ object Router {
 
   // @formatter:off
   case class SendChannelQuery(chainHash: ByteVector32, remoteNodeId: PublicKey, to: ActorRef, replacePrevious: Boolean, flags_opt: Option[QueryChannelRangeTlv]) extends RemoteTypes
-  case object GetNetworkStats
-  case class GetNetworkStatsResponse(stats: Option[NetworkStats])
   case object GetRoutingState
   case class RoutingState(channels: Iterable[PublicChannel], nodes: Iterable[NodeAnnouncement])
   case object GetRoutingStateStreaming extends RemoteTypes
@@ -591,7 +567,6 @@ object Router {
 
   case class Data(nodes: Map[PublicKey, NodeAnnouncement],
                   channels: SortedMap[ShortChannelId, PublicChannel],
-                  stats: Option[NetworkStats],
                   stash: Stash,
                   rebroadcast: Rebroadcast,
                   awaiting: Map[ChannelAnnouncement, Seq[RemoteGossip]], // note: this is a seq because we want to preserve order: first actor is the one who we need to send a tcp-ack when validation is done
@@ -607,7 +582,6 @@ object Router {
 
   case object TickBroadcast
   case object TickPruneStaleChannels
-  case object TickComputeNetworkStats
   // @formatter:on
 
   def getDesc(u: ChannelUpdate, channel: ChannelAnnouncement): ChannelDesc = {

eclair-core/src/test/scala/fr/acinq/eclair/EclairImplSpec.scala

@@ -36,8 +36,8 @@ import fr.acinq.eclair.payment.receive.PaymentHandler
 import fr.acinq.eclair.payment.relay.Relayer.RelayFees
 import fr.acinq.eclair.payment.send.PaymentInitiator.{SendPaymentToNode, SendPaymentToRoute, SendSpontaneousPayment}
 import fr.acinq.eclair.router.RouteCalculationSpec.makeUpdateShort
-import fr.acinq.eclair.router.Router.{GetNetworkStats, GetNetworkStatsResponse, PredefinedNodeRoute, PublicChannel}
-import fr.acinq.eclair.router.{Announcements, NetworkStats, Router, Stats}
+import fr.acinq.eclair.router.Router.{PredefinedNodeRoute, PublicChannel}
+import fr.acinq.eclair.router.{Announcements, Router}
 import fr.acinq.eclair.wire.internal.channel.ChannelCodecsSpec
 import fr.acinq.eclair.wire.protocol.{ChannelUpdate, Color, NodeAnnouncement}
 import org.mockito.Mockito
@@ -264,28 +264,6 @@ class EclairImplSpec extends TestKitBaseClass with FixtureAnyFunSuiteLike with I
     })
   }
 
-  test("router returns Network Stats") { f =>
-    import f._
-
-    val capStat = Stats(30 sat, 12 sat, 14 sat, 20 sat, 40 sat, 46 sat, 48 sat)
-    val cltvStat = Stats(CltvExpiryDelta(32), CltvExpiryDelta(11), CltvExpiryDelta(13), CltvExpiryDelta(22), CltvExpiryDelta(42), CltvExpiryDelta(51), CltvExpiryDelta(53))
-    val feeBaseStat = Stats(32 msat, 11 msat, 13 msat, 22 msat, 42 msat, 51 msat, 53 msat)
-    val feePropStat = Stats(32L, 11L, 13L, 22L, 42L, 51L, 53L)
-    val eclair = new EclairImpl(kit)
-    val fResp = eclair.networkStats()
-    f.router.expectMsg(GetNetworkStats)
-
-    f.router.reply(GetNetworkStatsResponse(Some(new NetworkStats(1, 2, capStat, cltvStat, feeBaseStat, feePropStat))))
-
-    awaitCond({
-      fResp.value match {
-        case Some(Success(Some(res))) => res.channels == 1
-        case _ => false
-      }
-    })
-
-  }
-
   test("close and forceclose should work both with channelId and shortChannelId") { f =>
     import f._
 

eclair-core/src/test/scala/fr/acinq/eclair/TestConstants.scala

@@ -168,7 +168,6 @@ object TestConstants {
       routerConf = RouterConf(
         channelExcludeDuration = 60 seconds,
         routerBroadcastInterval = 5 seconds,
-        networkStatsRefreshInterval = 1 hour,
         requestNodeAnnouncements = true,
         encodingType = EncodingType.COMPRESSED_ZLIB,
         channelRangeChunkSize = 20,
@@ -295,7 +294,6 @@ object TestConstants {
       routerConf = RouterConf(
         channelExcludeDuration = 60 seconds,
         routerBroadcastInterval = 5 seconds,
-        networkStatsRefreshInterval = 1 hour,
         requestNodeAnnouncements = true,
         encodingType = EncodingType.UNCOMPRESSED,
         channelRangeChunkSize = 20,