Store published txs in AuditDb (#1976)

We previously computed the on-chain fees paid by us after the fact, when
receiving a notification that a transaction was confirmed. This worked
because lightning transactions had a single input, which we stored in
our DB to allow us to compute the fee.

With anchor outputs, this mechanism doesn't work anymore. Some txs have
their fees paid by a child tx, and may have more than one input.

We completely change our model to store every transaction we publish,
along with the fee we're paying for this transaction. We then separately
store every transaction that confirms, which lets us join these two data
sets to compute how much on-chain fees we paid.

This has the added benefit that we can now audit every transaction that
we tried to publish, which lets node operators audit the anchor outputs
internal RBF mechanism and all the on-chain footprint of a given channel.

docs/release-notes/eclair-vnext.md

@@ -116,6 +116,16 @@ We still support receiving non-segwit remote scripts, but will force-close if th
 
 See the [spec discussions](https://github.com/lightningnetwork/lightning-rfc/pull/894) for more details.
 
+### Audit trail for published transactions
+
+Eclair now records every transaction it publishes in the `audit` database, in a new `transactions_published` table.
+It also stores confirmed transactions that have an impact on existing channels (including transactions made by your peer) in a new `transactions_confirmed` table.
+
+This lets you audit the complete on-chain footprint of your channels and the on-chain fees paid.
+This information is exposed through the `networkfees` API (which was already available in previous versions).
+
+We removed the previous `network_fees` table which achieved the same result but contained less details.
+
 ### Sample GUI removed
 
 We previously included code for a sample GUI: `eclair-node-gui`.

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelEvents.scala

@@ -60,9 +60,12 @@ case class ChannelSignatureReceived(channel: ActorRef, commitments: Commitments)
 
 case class ChannelErrorOccurred(channel: ActorRef, channelId: ByteVector32, remoteNodeId: PublicKey, data: ChannelData, error: ChannelOpenError, isFatal: Boolean) extends ChannelEvent
 
-case class NetworkFeePaid(channel: ActorRef, remoteNodeId: PublicKey, channelId: ByteVector32, tx: Transaction, fee: Satoshi, txType: String) extends ChannelEvent
+// NB: the fee should be set to 0 when we're not paying it.
+case class TransactionPublished(channelId: ByteVector32, remoteNodeId: PublicKey, tx: Transaction, miningFee: Satoshi, desc: String) extends ChannelEvent
 
-// NB: this event is only sent when the channel is available
+case class TransactionConfirmed(channelId: ByteVector32, remoteNodeId: PublicKey, tx: Transaction) extends ChannelEvent
+
+// NB: this event is only sent when the channel is available.
 case class AvailableBalanceChanged(channel: ActorRef, channelId: ByteVector32, shortChannelId: ShortChannelId, commitments: AbstractCommitments) extends ChannelEvent
 
 case class ChannelPersisted(channel: ActorRef, remoteNodeId: PublicKey, channelId: ByteVector32, data: HasCommitments) extends ChannelEvent

eclair-core/src/main/scala/fr/acinq/eclair/channel/Helpers.scala

@@ -36,8 +36,8 @@ import fr.acinq.eclair.transactions._
 import fr.acinq.eclair.wire.protocol._
 import scodec.bits.ByteVector
 
-import scala.concurrent.{Await, ExecutionContext}
 import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext}
 import scala.util.{Failure, Success, Try}
 
 /**
@@ -565,6 +565,13 @@ object Helpers {
       }
     }
 
+    /** Compute the fee paid by a commitment transaction. */
+    def commitTxFee(commitInput: InputInfo, commitTx: Transaction, isFunder: Boolean): Satoshi = {
+      require(commitTx.txIn.size == 1, "transaction must have only one input")
+      require(commitTx.txIn.exists(txIn => txIn.outPoint == commitInput.outPoint), "transaction must spend the funding output")
+      if (isFunder) commitInput.txOut.amount - commitTx.txOut.map(_.amount).sum else 0 sat
+    }
+
     /**
      * Claim all the HTLCs that we've received from our current commit tx. This will be done using 2nd stage HTLC transactions.
      *
@@ -668,18 +675,17 @@ object Helpers {
      * @return a list of transactions (one per output of the commit tx that we can claim)
      */
     def claimRemoteCommitTxOutputs(keyManager: ChannelKeyManager, commitments: Commitments, remoteCommit: RemoteCommit, tx: Transaction, feeEstimator: FeeEstimator, feeTargets: FeeTargets)(implicit log: LoggingAdapter): RemoteCommitPublished = {
-      import commitments.{channelConfig, channelFeatures, commitInput, localParams, remoteParams}
       require(remoteCommit.txid == tx.txid, "txid mismatch, provided tx is not the current remote commit tx")
-      val (remoteCommitTx, _) = Commitments.makeRemoteTxs(keyManager, channelConfig, channelFeatures, remoteCommit.index, localParams, remoteParams, commitInput, remoteCommit.remotePerCommitmentPoint, remoteCommit.spec)
+      val (remoteCommitTx, _) = Commitments.makeRemoteTxs(keyManager, commitments.channelConfig, commitments.channelFeatures, remoteCommit.index, commitments.localParams, commitments.remoteParams, commitments.commitInput, remoteCommit.remotePerCommitmentPoint, remoteCommit.spec)
       require(remoteCommitTx.tx.txid == tx.txid, "txid mismatch, cannot recompute the current remote commit tx")
-      val channelKeyPath = keyManager.keyPath(localParams, channelConfig)
-      val localFundingPubkey = keyManager.fundingPublicKey(localParams.fundingKeyPath).publicKey
+      val channelKeyPath = keyManager.keyPath(commitments.localParams, commitments.channelConfig)
+      val localFundingPubkey = keyManager.fundingPublicKey(commitments.localParams.fundingKeyPath).publicKey
       val localHtlcPubkey = Generators.derivePubKey(keyManager.htlcPoint(channelKeyPath).publicKey, remoteCommit.remotePerCommitmentPoint)
-      val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, remoteCommit.remotePerCommitmentPoint)
+      val remoteHtlcPubkey = Generators.derivePubKey(commitments.remoteParams.htlcBasepoint, remoteCommit.remotePerCommitmentPoint)
       val remoteRevocationPubkey = Generators.revocationPubKey(keyManager.revocationPoint(channelKeyPath).publicKey, remoteCommit.remotePerCommitmentPoint)
-      val remoteDelayedPaymentPubkey = Generators.derivePubKey(remoteParams.delayedPaymentBasepoint, remoteCommit.remotePerCommitmentPoint)
+      val remoteDelayedPaymentPubkey = Generators.derivePubKey(commitments.remoteParams.delayedPaymentBasepoint, remoteCommit.remotePerCommitmentPoint)
       val localPaymentPubkey = Generators.derivePubKey(keyManager.paymentPoint(channelKeyPath).publicKey, remoteCommit.remotePerCommitmentPoint)
-      val outputs = makeCommitTxOutputs(!localParams.isFunder, remoteParams.dustLimit, remoteRevocationPubkey, localParams.toSelfDelay, remoteDelayedPaymentPubkey, localPaymentPubkey, remoteHtlcPubkey, localHtlcPubkey, remoteParams.fundingPubKey, localFundingPubkey, remoteCommit.spec, commitments.commitmentFormat)
+      val outputs = makeCommitTxOutputs(!commitments.localParams.isFunder, commitments.remoteParams.dustLimit, remoteRevocationPubkey, commitments.localParams.toSelfDelay, remoteDelayedPaymentPubkey, localPaymentPubkey, remoteHtlcPubkey, localHtlcPubkey, commitments.remoteParams.fundingPubKey, localFundingPubkey, remoteCommit.spec, commitments.commitmentFormat)
 
       // we need to use a rather high fee for htlc-claim because we compete with the counterparty
       val feeratePerKwHtlc = feeEstimator.getFeeratePerKw(target = 2)
@@ -691,7 +697,7 @@ object Helpers {
       val htlcTxs: Map[OutPoint, Option[ClaimHtlcTx]] = remoteCommit.spec.htlcs.collect {
         case OutgoingHtlc(add: UpdateAddHtlc) =>
           generateTx("claim-htlc-success") {
-            Transactions.makeClaimHtlcSuccessTx(remoteCommitTx.tx, outputs, localParams.dustLimit, localHtlcPubkey, remoteHtlcPubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc, commitments.commitmentFormat)
+            Transactions.makeClaimHtlcSuccessTx(remoteCommitTx.tx, outputs, commitments.localParams.dustLimit, localHtlcPubkey, remoteHtlcPubkey, remoteRevocationPubkey, commitments.localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc, commitments.commitmentFormat)
           }.map(claimHtlcTx => {
             if (preimages.contains(add.paymentHash)) {
               // incoming htlc for which we have the preimage: we can spend it immediately
@@ -708,7 +714,7 @@ object Helpers {
         case IncomingHtlc(add: UpdateAddHtlc) =>
           // outgoing htlc: they may or may not have the preimage, the only thing to do is try to get back our funds after timeout
           generateTx("claim-htlc-timeout") {
-            Transactions.makeClaimHtlcTimeoutTx(remoteCommitTx.tx, outputs, localParams.dustLimit, localHtlcPubkey, remoteHtlcPubkey, remoteRevocationPubkey, localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc, commitments.commitmentFormat)
+            Transactions.makeClaimHtlcTimeoutTx(remoteCommitTx.tx, outputs, commitments.localParams.dustLimit, localHtlcPubkey, remoteHtlcPubkey, remoteRevocationPubkey, commitments.localParams.defaultFinalScriptPubKey, add, feeratePerKwHtlc, commitments.commitmentFormat)
           }.map(claimHtlcTx => {
             claimHtlcTx.input.outPoint -> generateTx("claim-htlc-timeout") {
               val sig = keyManager.sign(claimHtlcTx, keyManager.htlcPoint(channelKeyPath), remoteCommit.remotePerCommitmentPoint, TxOwner.Local, commitments.commitmentFormat)
@@ -726,7 +732,7 @@ object Helpers {
         }
       ).flatten
 
-      if (channelFeatures.paysDirectlyToWallet) {
+      if (commitments.channelFeatures.paysDirectlyToWallet) {
         RemoteCommitPublished(
           commitTx = tx,
           claimMainOutputTx = None,
@@ -1255,64 +1261,6 @@ object Helpers {
       irrevocablySpent.contains(input)
     }
 
-    /**
-     * This helper function returns the fee paid by the given transaction.
-     * It relies on the current channel data to find the parent tx and compute the fee, and also provides a description.
-     *
-     * @param tx a tx for which we want to compute the fee
-     * @param d  current channel data
-     * @return if the parent tx is found, a tuple (fee, description)
-     */
-    def networkFeePaid(tx: Transaction, d: DATA_CLOSING): Option[(Satoshi, String)] = {
-      val isCommitTx = tx.txIn.map(_.outPoint).contains(d.commitments.commitInput.outPoint)
-      // only the funder pays the fee for the commit tx, but 2nd-stage and 3rd-stage tx fees are paid by their recipients
-      // we can compute the fees only for transactions with a single parent for which we know the output amount
-      if (tx.txIn.size == 1 && (d.commitments.localParams.isFunder || !isCommitTx)) {
-        // we build a map with all known txs (that's not particularly efficient, but it doesn't really matter)
-        val txs: Map[ByteVector32, (Transaction, String)] = (
-          d.mutualClosePublished.map(_.tx -> "mutual") ++
-            d.localCommitPublished.map(_.commitTx).map(_ -> "local-commit").toSeq ++
-            d.localCommitPublished.flatMap(_.claimMainDelayedOutputTx).map(_.tx -> "local-main-delayed") ++
-            d.localCommitPublished.toSeq.flatMap(_.htlcTxs.values).flatten.map {
-              case htlcTx: HtlcSuccessTx => htlcTx.tx -> "local-htlc-success"
-              case htlcTx: HtlcTimeoutTx => htlcTx.tx -> "local-htlc-timeout"
-            } ++
-            d.localCommitPublished.toSeq.flatMap(_.claimHtlcDelayedTxs).map(_.tx -> "local-htlc-delayed") ++
-            d.remoteCommitPublished.map(_.commitTx).map(_ -> "remote-commit") ++
-            d.remoteCommitPublished.toSeq.flatMap(_.claimMainOutputTx).map(_.tx -> "remote-main") ++
-            d.remoteCommitPublished.toSeq.flatMap(_.claimHtlcTxs.values).flatten.map {
-              case htlcTx: ClaimHtlcSuccessTx => htlcTx.tx -> "remote-htlc-success"
-              case htlcTx: ClaimHtlcTimeoutTx => htlcTx.tx -> "remote-htlc-timeout"
-            } ++
-            d.nextRemoteCommitPublished.map(_.commitTx).map(_ -> "remote-commit") ++
-            d.nextRemoteCommitPublished.toSeq.flatMap(_.claimMainOutputTx).map(_.tx -> "remote-main") ++
-            d.nextRemoteCommitPublished.toSeq.flatMap(_.claimHtlcTxs.values).flatten.map {
-              case htlcTx: ClaimHtlcSuccessTx => htlcTx.tx -> "remote-htlc-success"
-              case htlcTx: ClaimHtlcTimeoutTx => htlcTx.tx -> "remote-htlc-timeout"
-            } ++
-            d.revokedCommitPublished.map(_.commitTx).map(_ -> "revoked-commit") ++
-            d.revokedCommitPublished.flatMap(_.claimMainOutputTx).map(_.tx -> "revoked-main") ++
-            d.revokedCommitPublished.flatMap(_.mainPenaltyTx).map(_.tx -> "revoked-main-penalty") ++
-            d.revokedCommitPublished.flatMap(_.htlcPenaltyTxs).map(_.tx -> "revoked-htlc-penalty") ++
-            d.revokedCommitPublished.flatMap(_.claimHtlcDelayedPenaltyTxs).map(_.tx -> "revoked-htlc-penalty-delayed")
-          )
-          .map { case (tx, desc) => tx.txid -> (tx, desc) } // will allow easy lookup of parent transaction
-          .toMap
-
-        txs.get(tx.txid).flatMap {
-          case (_, desc) =>
-            val parentTxOut_opt = if (isCommitTx) {
-              Some(d.commitments.commitInput.txOut)
-            } else {
-              val outPoint = tx.txIn.head.outPoint
-              txs.get(outPoint.txid).map { case (parent, _) => parent.txOut(outPoint.index.toInt) }
-            }
-            parentTxOut_opt.map(parentTxOut => parentTxOut.amount - tx.txOut.map(_.amount).sum).map(_ -> desc)
-        }
-      } else {
-        None
-      }
-    }
   }
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/MempoolTxMonitor.scala

@@ -19,11 +19,12 @@ package fr.acinq.eclair.channel.publish
 import akka.actor.typed.eventstream.EventStream
 import akka.actor.typed.scaladsl.{ActorContext, Behaviors}
 import akka.actor.typed.{ActorRef, Behavior}
-import fr.acinq.bitcoin.{OutPoint, Transaction}
+import fr.acinq.bitcoin.{ByteVector32, OutPoint, Satoshi, Transaction}
 import fr.acinq.eclair.NodeParams
 import fr.acinq.eclair.blockchain.CurrentBlockCount
 import fr.acinq.eclair.blockchain.bitcoind.rpc.BitcoinCoreClient
 import fr.acinq.eclair.channel.publish.TxPublisher.{TxPublishLogContext, TxRejectedReason}
+import fr.acinq.eclair.channel.{TransactionConfirmed, TransactionPublished}
 
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success}
@@ -36,7 +37,7 @@ object MempoolTxMonitor {
 
   // @formatter:off
   sealed trait Command
-  case class Publish(replyTo: ActorRef[TxResult], tx: Transaction, input: OutPoint) extends Command
+  case class Publish(replyTo: ActorRef[TxResult], tx: Transaction, input: OutPoint, desc: String, fee: Satoshi) extends Command
   private case object PublishOk extends Command
   private case class PublishFailed(reason: Throwable) extends Command
   private case class InputStatus(spentConfirmed: Boolean, spentUnconfirmed: Boolean) extends Command
@@ -57,34 +58,35 @@ object MempoolTxMonitor {
   def apply(nodeParams: NodeParams, bitcoinClient: BitcoinCoreClient, loggingInfo: TxPublishLogContext): Behavior[Command] = {
     Behaviors.setup { context =>
       Behaviors.withMdc(loggingInfo.mdc()) {
-        new MempoolTxMonitor(nodeParams, bitcoinClient, context).start()
+        new MempoolTxMonitor(nodeParams, bitcoinClient, loggingInfo, context).start()
       }
     }
   }
 
 }
 
-private class MempoolTxMonitor(nodeParams: NodeParams, bitcoinClient: BitcoinCoreClient, context: ActorContext[MempoolTxMonitor.Command])(implicit ec: ExecutionContext = ExecutionContext.Implicits.global) {
+private class MempoolTxMonitor(nodeParams: NodeParams, bitcoinClient: BitcoinCoreClient, loggingInfo: TxPublishLogContext, context: ActorContext[MempoolTxMonitor.Command])(implicit ec: ExecutionContext = ExecutionContext.Implicits.global) {
 
   import MempoolTxMonitor._
 
   private val log = context.log
 
   def start(): Behavior[Command] = {
     Behaviors.receiveMessagePartial {
-      case Publish(replyTo, tx, input) => publish(replyTo, tx, input)
+      case Publish(replyTo, tx, input, desc, fee) => publish(replyTo, tx, input, desc, fee)
       case Stop => Behaviors.stopped
     }
   }
 
-  def publish(replyTo: ActorRef[TxResult], tx: Transaction, input: OutPoint): Behavior[Command] = {
+  def publish(replyTo: ActorRef[TxResult], tx: Transaction, input: OutPoint, desc: String, fee: Satoshi): Behavior[Command] = {
     context.pipeToSelf(bitcoinClient.publishTransaction(tx)) {
       case Success(_) => PublishOk
       case Failure(reason) => PublishFailed(reason)
     }
     Behaviors.receiveMessagePartial {
       case PublishOk =>
         log.debug("txid={} was successfully published, waiting for confirmation...", tx.txid)
+        context.system.eventStream ! EventStream.Publish(TransactionPublished(loggingInfo.channelId_opt.getOrElse(ByteVector32.Zeroes), loggingInfo.remoteNodeId, tx, fee, desc))
         waitForConfirmation(replyTo, tx, input)
       case PublishFailed(reason) if reason.getMessage.contains("rejecting replacement") =>
         log.info("could not publish tx: a conflicting mempool transaction is already in the mempool")
@@ -133,6 +135,7 @@ private class MempoolTxMonitor(nodeParams: NodeParams, bitcoinClient: BitcoinCor
         }
         if (nodeParams.minDepthBlocks <= confirmations) {
           log.info("txid={} has reached min depth", tx.txid)
+          context.system.eventStream ! EventStream.Publish(TransactionConfirmed(loggingInfo.channelId_opt.getOrElse(ByteVector32.Zeroes), loggingInfo.remoteNodeId, tx))
           sendResult(replyTo, TxConfirmed, Some(messageAdapter))
         } else {
           Behaviors.same

eclair-core/src/main/scala/fr/acinq/eclair/channel/publish/RawTxPublisher.scala

@@ -119,7 +119,7 @@ private class RawTxPublisher(nodeParams: NodeParams,
 
   def publish(replyTo: ActorRef[TxPublisher.PublishTxResult], cmd: TxPublisher.PublishRawTx): Behavior[Command] = {
     val txMonitor = context.spawn(MempoolTxMonitor(nodeParams, bitcoinClient, loggingInfo), "mempool-tx-monitor")
-    txMonitor ! MempoolTxMonitor.Publish(context.messageAdapter[MempoolTxMonitor.TxResult](WrappedTxResult), cmd.tx, cmd.input)
+    txMonitor ! MempoolTxMonitor.Publish(context.messageAdapter[MempoolTxMonitor.TxResult](WrappedTxResult), cmd.tx, cmd.input, cmd.desc, cmd.fee)
     Behaviors.receiveMessagePartial {
       case WrappedTxResult(MempoolTxMonitor.TxConfirmed) => sendResult(replyTo, TxPublisher.TxConfirmed(cmd, cmd.tx))
       case WrappedTxResult(MempoolTxMonitor.TxRejected(reason)) => sendResult(replyTo, TxPublisher.TxRejected(loggingInfo.id, cmd, reason))