cli: add retry for deals stuck in Publish with no funds

.circleci/config.yml

@@ -829,6 +829,11 @@ workflows:
           suite: itest-deals_publish
           target: "./itests/deals_publish_test.go"
 
+      - test:
+          name: test-itest-deals_retry_deal_no_funds
+          suite: itest-deals_retry_deal_no_funds
+          target: "./itests/deals_retry_deal_no_funds_test.go"
+
       - test:
           name: test-itest-deals
           suite: itest-deals

api/api_storage.go

@@ -166,6 +166,7 @@ type StorageMiner interface {
 	MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write
 	MarketPendingDeals(ctx context.Context) (PendingDealInfo, error)                                                             //perm:write
 	MarketPublishPendingDeals(ctx context.Context) error                                                                         //perm:admin
+	MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error                                                           //perm:admin
 
 	// DagstoreListShards returns information about all shards known to the
 	// DAG store. Only available on nodes running the markets subsystem.

cmd/lotus-miner/market.go

@@ -352,6 +352,7 @@ var storageDealsCmd = &cli.Command{
 		resetBlocklistCmd,
 		setSealDurationCmd,
 		dealsPendingPublish,
+		dealsRetryPublish,
 	},
 }
 
@@ -910,6 +911,34 @@ var dealsPendingPublish = &cli.Command{
 	},
 }
 
+var dealsRetryPublish = &cli.Command{
+	Name:  "retry-publish",
+	Usage: "retry publishing a deal",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name: "proposal-cid",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		cid, err := cid.Decode(cctx.String("proposal-cid"))
+		if err != nil {
+			return err
+		}
+		if err := api.MarketRetryPublishDeal(ctx, cid); err != nil {
+			return xerrors.Errorf("retrying publishing deal: %w", err)
+		}
+		fmt.Println("retried to publish deal")
+		return nil
+	},
+}
+
 func listDealsWithJSON(cctx *cli.Context) error {
 	node, closer, err := lcli.GetMarketsAPI(cctx)
 	if err != nil {

documentation/en/api-v0-methods-miner.md

@@ -61,6 +61,7 @@
   * [MarketPendingDeals](#MarketPendingDeals)
   * [MarketPublishPendingDeals](#MarketPublishPendingDeals)
   * [MarketRestartDataTransfer](#MarketRestartDataTransfer)
+  * [MarketRetryPublishDeal](#MarketRetryPublishDeal)
   * [MarketSetAsk](#MarketSetAsk)
   * [MarketSetRetrievalAsk](#MarketSetRetrievalAsk)
 * [Mining](#Mining)
@@ -949,6 +950,22 @@ Inputs:
 
 Response: `{}`
 
+### MarketRetryPublishDeal
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  {
+    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+  }
+]
+```
+
+Response: `{}`
+
 ### MarketSetAsk
 
 

documentation/en/cli-lotus-miner.md

@@ -629,6 +629,7 @@ COMMANDS:
    reset-blocklist    Remove all entries from the miner's piece CID blocklist
    set-seal-duration  Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected.
    pending-publish    list deals waiting in publish queue
+   retry-publish      retry publishing a deal
    help, h            Shows a list of commands or help for one command
 
 OPTIONS:
@@ -825,6 +826,20 @@ OPTIONS:
    
 ```
 
+### lotus-miner storage-deals retry-publish
+```
+NAME:
+   lotus-miner storage-deals retry-publish - retry publishing a deal
+
+USAGE:
+   lotus-miner storage-deals retry-publish [command options] [arguments...]
+
+OPTIONS:
+   --proposal-cid value  
+   --help, -h            show help (default: false)
+   
+```
+
 ## lotus-miner retrieval-deals
 ```
 NAME:

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v1.11.1
 	github.com/filecoin-project/go-fil-commcid v0.1.0
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
-	github.com/filecoin-project/go-fil-markets v1.13.1
+	github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-padreader v0.0.0-20210723183308-812a16dc01b1
 	github.com/filecoin-project/go-paramfetch v0.0.2

go.sum

@@ -313,8 +313,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo=
 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.13.1 h1:KjarxgKp/RN4iYXT2pMcMq6veIa1guGJMoVtnwru4BQ=
-github.com/filecoin-project/go-fil-markets v1.13.1/go.mod h1:58OjtsWtDt3xlN1QLmgDQxtfCDtDS4RIyHepIUbqXhM=
+github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb h1:8e9XhhvYCUS91GeP4HXj6rH2ySShLuWRDkwff1CFha0=
+github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb/go.mod h1:58OjtsWtDt3xlN1QLmgDQxtfCDtDS4RIyHepIUbqXhM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=

itests/deals_retry_deal_no_funds_test.go

@@ -0,0 +1,107 @@
+package itests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/itests/kit"
+	"github.com/filecoin-project/lotus/markets/storageadapter"
+	"github.com/filecoin-project/lotus/node"
+	"github.com/filecoin-project/lotus/node/config"
+	"github.com/filecoin-project/lotus/node/modules"
+	"github.com/filecoin-project/lotus/storage"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	publishPeriod  = 1 * time.Second
+	maxDealsPerMsg = uint64(2) // Set max deals per publish deals message to 2
+
+	blockTime = 10 * time.Millisecond
+)
+
+func TestDealsRetryLackOfFunds(t *testing.T) {
+	ctx := context.Background()
+	oldDelay := policy.GetPreCommitChallengeDelay()
+	policy.SetPreCommitChallengeDelay(5)
+
+	t.Cleanup(func() {
+		policy.SetPreCommitChallengeDelay(oldDelay)
+	})
+
+	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1)
+	kit.QuietMiningLogs()
+
+	// Allow 8MB sectors
+	eightMBSectorsOpt := kit.SectorSize(8 << 20)
+
+	publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1)
+	require.NoError(t, err)
+
+	opts := node.Options(
+		node.Override(new(*storageadapter.DealPublisher),
+			storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{
+				Period:         publishPeriod,
+				MaxDealsPerMsg: maxDealsPerMsg,
+			}),
+		),
+		node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{
+			DealPublishControl: []string{
+				publishStorageDealKey.Address.String(),
+			},
+			DisableOwnerFallback:  true,
+			DisableWorkerFallback: true,
+		})),
+	)
+
+	minerFullNode, clientFullNode, miner, ens := kit.EnsembleTwoOne(t, kit.Account(publishStorageDealKey, types.NewInt(1020000000000)), kit.ConstructorOpts(opts), kit.MockProofs(), eightMBSectorsOpt)
+
+	//TODO: this fails slightly differently - handle this case as well
+	//minerFullNode, clientFullNode, miner, ens := kit.EnsembleTwoOne(t, kit.Account(publishStorageDealKey, types.NewInt(1)), kit.ConstructorOpts(opts), kit.MockProofs(), eightMBSectorsOpt)
+
+	kit.QuietMiningLogs()
+
+	ens.
+		Start().
+		InterconnectAll().
+		BeginMining(blockTime)
+
+	_, err = minerFullNode.WalletImport(ctx, &publishStorageDealKey.KeyInfo)
+	require.NoError(t, err)
+
+	miner.SetControlAddresses(publishStorageDealKey.Address)
+
+	dh := kit.NewDealHarness(t, clientFullNode, miner, miner)
+
+	res, _ := clientFullNode.CreateImportFile(ctx, 0, 4<<20) // 4MiB file.
+	list, err := clientFullNode.ClientListImports(ctx)
+	require.NoError(t, err)
+	require.Len(t, list, 1)
+	require.Equal(t, res.Root, *list[0].Root)
+
+	dp := dh.DefaultStartDealParams()
+	dp.Data.Root = res.Root
+	dp.FastRetrieval = true
+	dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price.
+	deal := dh.StartDeal(ctx, dp)
+
+	propcid := *deal
+
+	go func() {
+		time.Sleep(20 * time.Second)
+
+		kit.SendFunds(ctx, t, minerFullNode, publishStorageDealKey.Address, types.FromFil(1))
+
+		err := miner.MarketRetryPublishDeal(ctx, propcid)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	dh.WaitDealSealed(ctx, deal, false, false, nil)
+}

itests/kit/deals.go

@@ -216,7 +216,7 @@ func (dh *DealHarness) ExpectDealFailure(ctx context.Context, deal *cid.Cid, err
 		}
 
 		dh.t.Logf("Deal %d state: client:%s provider:%s\n", di.DealID, storagemarket.DealStates[di.State], storagemarket.DealStates[minerState])
-		time.Sleep(time.Second / 2)
+		time.Sleep(time.Second)
 	}
 }
 

itests/kit/funds.go

@@ -32,6 +32,19 @@ func SendFunds(ctx context.Context, t *testing.T, sender *TestFullNode, recipien
 	sender.WaitMsg(ctx, sm.Cid())
 }
 
+func SendFundsFromAddr(ctx context.Context, t *testing.T, sender *TestFullNode, senderAddr address.Address, recipient address.Address, amount abi.TokenAmount) {
+	msg := &types.Message{
+		From:  senderAddr,
+		To:    recipient,
+		Value: amount,
+	}
+
+	sm, err := sender.MpoolPushMessage(ctx, msg, nil)
+	require.NoError(t, err)
+
+	sender.WaitMsg(ctx, sm.Cid())
+}
+
 func (f *TestFullNode) WaitMsg(ctx context.Context, msg cid.Cid) {
 	res, err := f.StateWaitMsg(ctx, msg, 3, api.LookbackNoLimit, true)
 	require.NoError(f.t, err)

node/impl/full/mpool.go

@@ -3,6 +3,7 @@ package full
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
@@ -177,6 +178,8 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 		return nil, xerrors.Errorf("mpool push: getting origin balance: %w", err)
 	}
 
+	fmt.Println("=== anton b balance: ", b)
+
 	if b.LessThan(msg.Value) {
 		return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value)
 	}

node/impl/storminer.go

@@ -557,6 +557,10 @@ func (sm *StorageMinerAPI) MarketPendingDeals(ctx context.Context) (api.PendingD
 	return sm.DealPublisher.PendingDeals(), nil
 }
 
+func (sm *StorageMinerAPI) MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error {
+	return sm.StorageProvider.RetryDealPublishing(propcid)
+}
+
 func (sm *StorageMinerAPI) MarketPublishPendingDeals(ctx context.Context) error {
 	sm.DealPublisher.ForcePublishPendingDeals()
 	return nil