diff --git a/.circleci/config.yml b/.circleci/config.yml index 222f14d5099..30f2d5c01fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -825,6 +825,11 @@ workflows: suite: itest-deals_publish target: "./itests/deals_publish_test.go" + - test: + name: test-itest-deals_retry_deal_no_funds + suite: itest-deals_retry_deal_no_funds + target: "./itests/deals_retry_deal_no_funds_test.go" + - test: name: test-itest-deals suite: itest-deals diff --git a/api/api_storage.go b/api/api_storage.go index 6ebee990882..8cca2aa5be5 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -166,6 +166,7 @@ type StorageMiner interface { MarketCancelDataTransfer(ctx context.Context, transferID datatransfer.TransferID, otherPeer peer.ID, isInitiator bool) error //perm:write MarketPendingDeals(ctx context.Context) (PendingDealInfo, error) //perm:write MarketPublishPendingDeals(ctx context.Context) error //perm:admin + MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error //perm:admin // DagstoreListShards returns information about all shards known to the // DAG store. Only available on nodes running the markets subsystem. diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 7a504cf7777..b36f19a7e1e 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -683,6 +683,8 @@ type StorageMinerStruct struct { MarketRestartDataTransfer func(p0 context.Context, p1 datatransfer.TransferID, p2 peer.ID, p3 bool) error `perm:"write"` + MarketRetryPublishDeal func(p0 context.Context, p1 cid.Cid) error `perm:"admin"` + MarketSetAsk func(p0 context.Context, p1 types.BigInt, p2 types.BigInt, p3 abi.ChainEpoch, p4 abi.PaddedPieceSize, p5 abi.PaddedPieceSize) error `perm:"admin"` MarketSetRetrievalAsk func(p0 context.Context, p1 *retrievalmarket.Ask) error `perm:"admin"` @@ -4020,6 +4022,17 @@ func (s *StorageMinerStub) MarketRestartDataTransfer(p0 context.Context, p1 data return ErrNotSupported } +func (s *StorageMinerStruct) MarketRetryPublishDeal(p0 context.Context, p1 cid.Cid) error { + if s.Internal.MarketRetryPublishDeal == nil { + return ErrNotSupported + } + return s.Internal.MarketRetryPublishDeal(p0, p1) +} + +func (s *StorageMinerStub) MarketRetryPublishDeal(p0 context.Context, p1 cid.Cid) error { + return ErrNotSupported +} + func (s *StorageMinerStruct) MarketSetAsk(p0 context.Context, p1 types.BigInt, p2 types.BigInt, p3 abi.ChainEpoch, p4 abi.PaddedPieceSize, p5 abi.PaddedPieceSize) error { if s.Internal.MarketSetAsk == nil { return ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 8837c745b4e..1c5765efe0d 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 6727576ef06..5bdcf035c3b 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 79e2aa85aa9..a35fad110ef 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/cmd/lotus-miner/market.go b/cmd/lotus-miner/market.go index c32f44b6a0a..d3da6f9b3a7 100644 --- a/cmd/lotus-miner/market.go +++ b/cmd/lotus-miner/market.go @@ -352,6 +352,7 @@ var storageDealsCmd = &cli.Command{ resetBlocklistCmd, setSealDurationCmd, dealsPendingPublish, + dealsRetryPublish, }, } @@ -910,6 +911,36 @@ var dealsPendingPublish = &cli.Command{ }, } +var dealsRetryPublish = &cli.Command{ + Name: "retry-publish", + Usage: "retry publishing a deal", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + api, closer, err := lcli.GetMarketsAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := lcli.ReqContext(cctx) + + propcid := cctx.Args().First() + fmt.Printf("retrying deal with proposal-cid: %s\n", propcid) + + cid, err := cid.Decode(propcid) + if err != nil { + return err + } + if err := api.MarketRetryPublishDeal(ctx, cid); err != nil { + return xerrors.Errorf("retrying publishing deal: %w", err) + } + fmt.Println("retried to publish deal") + return nil + }, +} + func listDealsWithJSON(cctx *cli.Context) error { node, closer, err := lcli.GetMarketsAPI(cctx) if err != nil { diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index dd7a1f88ea8..4d14bcb0e34 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -61,6 +61,7 @@ * [MarketPendingDeals](#MarketPendingDeals) * [MarketPublishPendingDeals](#MarketPublishPendingDeals) * [MarketRestartDataTransfer](#MarketRestartDataTransfer) + * [MarketRetryPublishDeal](#MarketRetryPublishDeal) * [MarketSetAsk](#MarketSetAsk) * [MarketSetRetrievalAsk](#MarketSetRetrievalAsk) * [Mining](#Mining) @@ -949,6 +950,22 @@ Inputs: Response: `{}` +### MarketRetryPublishDeal + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `{}` + ### MarketSetAsk diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 5a0888621d7..b77daf8003c 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -629,6 +629,7 @@ COMMANDS: reset-blocklist Remove all entries from the miner's piece CID blocklist set-seal-duration Set the expected time, in minutes, that you expect sealing sectors to take. Deals that start before this duration will be rejected. pending-publish list deals waiting in publish queue + retry-publish retry publishing a deal help, h Shows a list of commands or help for one command OPTIONS: @@ -825,6 +826,19 @@ OPTIONS: ``` +### lotus-miner storage-deals retry-publish +``` +NAME: + lotus-miner storage-deals retry-publish - retry publishing a deal + +USAGE: + lotus-miner storage-deals retry-publish [command options] + +OPTIONS: + --help, -h show help (default: false) + +``` + ## lotus-miner retrieval-deals ``` NAME: diff --git a/go.mod b/go.mod index b310b76fef6..0f352fd82c8 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.11.1 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.13.1 + github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.2 diff --git a/go.sum b/go.sum index 57cb969af37..89770ce957e 100644 --- a/go.sum +++ b/go.sum @@ -338,8 +338,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c= -github.com/filecoin-project/go-fil-markets v1.13.1 h1:KjarxgKp/RN4iYXT2pMcMq6veIa1guGJMoVtnwru4BQ= -github.com/filecoin-project/go-fil-markets v1.13.1/go.mod h1:58OjtsWtDt3xlN1QLmgDQxtfCDtDS4RIyHepIUbqXhM= +github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb h1:8e9XhhvYCUS91GeP4HXj6rH2ySShLuWRDkwff1CFha0= +github.com/filecoin-project/go-fil-markets v1.13.2-0.20211007101645-eebce51848eb/go.mod h1:58OjtsWtDt3xlN1QLmgDQxtfCDtDS4RIyHepIUbqXhM= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= diff --git a/itests/deals_retry_deal_no_funds_test.go b/itests/deals_retry_deal_no_funds_test.go new file mode 100644 index 00000000000..202d86b9fc1 --- /dev/null +++ b/itests/deals_retry_deal_no_funds_test.go @@ -0,0 +1,250 @@ +package itests + +import ( + "context" + "testing" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/filecoin-project/lotus/markets/storageadapter" + "github.com/filecoin-project/lotus/node" + "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/storage" + "github.com/stretchr/testify/require" +) + +var ( + publishPeriod = 1 * time.Second + maxDealsPerMsg = uint64(2) // Set max deals per publish deals message to 2 + + blockTime = 3 * time.Millisecond +) + +func TestDealsRetryLackOfFunds(t *testing.T) { + ctx := context.Background() + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + kit.QuietMiningLogs() + + // Allow 8MB sectors + eightMBSectorsOpt := kit.SectorSize(8 << 20) + + publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + require.NoError(t, err) + + opts := node.Options( + node.Override(new(*storageadapter.DealPublisher), + storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{ + Period: publishPeriod, + MaxDealsPerMsg: maxDealsPerMsg, + }), + ), + node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + DealPublishControl: []string{ + publishStorageDealKey.Address.String(), + }, + DisableOwnerFallback: true, + DisableWorkerFallback: true, + })), + ) + + publishStorageAccountFunds := types.NewInt(1020000000000) + minerFullNode, clientFullNode, miner, ens := kit.EnsembleTwoOne(t, kit.Account(publishStorageDealKey, publishStorageAccountFunds), kit.ConstructorOpts(opts), kit.MockProofs(), eightMBSectorsOpt) + + kit.QuietMiningLogs() + + ens. + Start(). + InterconnectAll(). + BeginMining(blockTime) + + _, err = minerFullNode.WalletImport(ctx, &publishStorageDealKey.KeyInfo) + require.NoError(t, err) + + miner.SetControlAddresses(publishStorageDealKey.Address) + + dh := kit.NewDealHarness(t, clientFullNode, miner, miner) + + res, _ := clientFullNode.CreateImportFile(ctx, 0, 4<<20) // 4MiB file. + list, err := clientFullNode.ClientListImports(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + require.Equal(t, res.Root, *list[0].Root) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.FastRetrieval = true + dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price. + deal := dh.StartDeal(ctx, dp) + + propcid := *deal + + go func() { + time.Sleep(3 * time.Second) + + kit.SendFunds(ctx, t, minerFullNode, publishStorageDealKey.Address, types.FromFil(1)) + + err := miner.MarketRetryPublishDeal(ctx, propcid) + if err != nil { + panic(err) + } + }() + + dh.WaitDealSealed(ctx, deal, false, false, nil) +} + +func TestDealsRetryLackOfFunds_blockInPublishDeal(t *testing.T) { + ctx := context.Background() + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + kit.QuietMiningLogs() + + // Allow 8MB sectors + eightMBSectorsOpt := kit.SectorSize(8 << 20) + + publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + require.NoError(t, err) + + opts := node.Options( + node.Override(new(*storageadapter.DealPublisher), + storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{ + Period: publishPeriod, + MaxDealsPerMsg: maxDealsPerMsg, + }), + ), + node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + DealPublishControl: []string{ + publishStorageDealKey.Address.String(), + }, + DisableOwnerFallback: true, + DisableWorkerFallback: true, + })), + ) + + publishStorageAccountFunds := types.NewInt(1020000000000) + minerFullNode, clientFullNode, miner, ens := kit.EnsembleTwoOne(t, kit.Account(publishStorageDealKey, publishStorageAccountFunds), kit.ConstructorOpts(opts), kit.MockProofs(), eightMBSectorsOpt) + + kit.QuietMiningLogs() + + ens. + Start(). + InterconnectAll(). + BeginMining(blockTime) + + _, err = minerFullNode.WalletImport(ctx, &publishStorageDealKey.KeyInfo) + require.NoError(t, err) + + miner.SetControlAddresses(publishStorageDealKey.Address) + + dh := kit.NewDealHarness(t, clientFullNode, miner, miner) + + res, _ := clientFullNode.CreateImportFile(ctx, 0, 4<<20) // 4MiB file. + list, err := clientFullNode.ClientListImports(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + require.Equal(t, res.Root, *list[0].Root) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.FastRetrieval = true + dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price. + deal := dh.StartDeal(ctx, dp) + + dealSealed := make(chan struct{}) + go func() { + dh.WaitDealSealedQuiet(ctx, deal, false, false, nil) + dealSealed <- struct{}{} + }() + + select { + case <-dealSealed: + t.Fatal("deal shouldn't have sealed") + case <-time.After(time.Second * 15): + } +} + +func TestDealsRetryLackOfFunds_belowLimit(t *testing.T) { + ctx := context.Background() + oldDelay := policy.GetPreCommitChallengeDelay() + policy.SetPreCommitChallengeDelay(5) + + t.Cleanup(func() { + policy.SetPreCommitChallengeDelay(oldDelay) + }) + + policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg8MiBV1) + kit.QuietMiningLogs() + + // Allow 8MB sectors + eightMBSectorsOpt := kit.SectorSize(8 << 20) + + publishStorageDealKey, err := wallet.GenerateKey(types.KTSecp256k1) + require.NoError(t, err) + + opts := node.Options( + node.Override(new(*storageadapter.DealPublisher), + storageadapter.NewDealPublisher(nil, storageadapter.PublishMsgConfig{ + Period: publishPeriod, + MaxDealsPerMsg: maxDealsPerMsg, + }), + ), + node.Override(new(*storage.AddressSelector), modules.AddressSelector(&config.MinerAddressConfig{ + DealPublishControl: []string{ + publishStorageDealKey.Address.String(), + }, + DisableOwnerFallback: true, + DisableWorkerFallback: true, + })), + ) + + publishStorageAccountFunds := types.NewInt(1) + minerFullNode, clientFullNode, miner, ens := kit.EnsembleTwoOne(t, kit.Account(publishStorageDealKey, publishStorageAccountFunds), kit.ConstructorOpts(opts), kit.MockProofs(), eightMBSectorsOpt) + + kit.QuietMiningLogs() + + ens. + Start(). + InterconnectAll(). + BeginMining(blockTime) + + _, err = minerFullNode.WalletImport(ctx, &publishStorageDealKey.KeyInfo) + require.NoError(t, err) + + miner.SetControlAddresses(publishStorageDealKey.Address) + + dh := kit.NewDealHarness(t, clientFullNode, miner, miner) + + res, _ := clientFullNode.CreateImportFile(ctx, 0, 4<<20) // 4MiB file. + list, err := clientFullNode.ClientListImports(ctx) + require.NoError(t, err) + require.Len(t, list, 1) + require.Equal(t, res.Root, *list[0].Root) + + dp := dh.DefaultStartDealParams() + dp.Data.Root = res.Root + dp.FastRetrieval = true + dp.EpochPrice = abi.NewTokenAmount(62500000) // minimum asking price. + deal := dh.StartDeal(ctx, dp) + + err = dh.ExpectDealFailure(ctx, deal, "actor balance less than needed") + if err != nil { + t.Fatal(err) + } +} diff --git a/itests/kit/deals.go b/itests/kit/deals.go index 1b1daa5e47d..4a9af69e627 100644 --- a/itests/kit/deals.go +++ b/itests/kit/deals.go @@ -177,6 +177,41 @@ loop: } } +// WaitDealSealedQuiet waits until the deal is sealed, without logging anything. +func (dh *DealHarness) WaitDealSealedQuiet(ctx context.Context, deal *cid.Cid, noseal, noSealStart bool, cb func()) { +loop: + for { + di, err := dh.client.ClientGetDealInfo(ctx, *deal) + require.NoError(dh.t, err) + + switch di.State { + case storagemarket.StorageDealAwaitingPreCommit, storagemarket.StorageDealSealing: + if noseal { + return + } + if !noSealStart { + dh.StartSealingWaiting(ctx) + } + case storagemarket.StorageDealProposalRejected: + dh.t.Fatal("deal rejected") + case storagemarket.StorageDealFailing: + dh.t.Fatal("deal failed") + case storagemarket.StorageDealError: + dh.t.Fatal("deal errored", di.Message) + case storagemarket.StorageDealActive: + break loop + } + + _, err = dh.market.MarketListIncompleteDeals(ctx) + require.NoError(dh.t, err) + + time.Sleep(time.Second / 2) + if cb != nil { + cb() + } + } +} + func (dh *DealHarness) ExpectDealFailure(ctx context.Context, deal *cid.Cid, errs string) error { for { di, err := dh.client.ClientGetDealInfo(ctx, *deal) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 4e970343e12..39baa97bf25 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -557,6 +557,10 @@ func (sm *StorageMinerAPI) MarketPendingDeals(ctx context.Context) (api.PendingD return sm.DealPublisher.PendingDeals(), nil } +func (sm *StorageMinerAPI) MarketRetryPublishDeal(ctx context.Context, propcid cid.Cid) error { + return sm.StorageProvider.RetryDealPublishing(propcid) +} + func (sm *StorageMinerAPI) MarketPublishPendingDeals(ctx context.Context) error { sm.DealPublisher.ForcePublishPendingDeals() return nil