feat: deals end to end test with restarts

testplans/lotus-soup/_compositions/baseline-docker-1-1-with-restarts.toml

@@ -0,0 +1,59 @@
+[metadata]
+  name = "lotus-soup"
+  author = ""
+
+[global]
+  plan = "lotus-soup"
+  case = "deals-e2e"
+  total_instances = 3
+  builder = "docker:go"
+  runner = "local:docker"
+
+[global.build]
+  selectors = ["testground"]
+
+[global.run_config]
+  exposed_ports = { pprof = "6060", node_rpc = "1234", miner_rpc = "2345" }
+
+[global.build_config]
+  enable_go_build_cache = true
+
+[global.run.test_params]
+  clients = "1"
+  miners = "1"
+  genesis_timestamp_offset = "0"
+  balance = "20000000" # These balances will work for maximum 100 nodes, as TotalFilecoin is 2B
+  sectors = "3"
+  random_beacon_type = "mock"
+  mining_mode = "natural"
+  bandwidth = "4MB"
+
+
+[[groups]]
+  id = "bootstrapper"
+  [groups.instances]
+    count = 1
+    percentage = 0.0
+  [groups.run]
+    [groups.run.test_params]
+      role = "bootstrapper"
+
+[[groups]]
+  id = "miners"
+  [groups.instances]
+    count = 1
+    percentage = 0.0
+  [groups.run]
+    [groups.run.test_params]
+      role = "miner"
+
+[[groups]]
+  id = "clients"
+  [groups.instances]
+    count = 1
+    percentage = 0.0
+  [groups.run]
+    [groups.run.test_params]
+      role = "client"
+      # Bounce the connection during push and pull requests
+      bounce_conn_data_transfers = "true"

testplans/lotus-soup/_compositions/baseline-docker-1-1.toml

@@ -23,7 +23,7 @@
   miners = "1"
   genesis_timestamp_offset = "0"
   balance = "20000000" # These balances will work for maximum 100 nodes, as TotalFilecoin is 2B
-  sectors = "10"
+  sectors = "3"
   random_beacon_type = "mock"
   mining_mode = "natural"
 

testplans/lotus-soup/deals_e2e.go

@@ -4,19 +4,19 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	mbig "math/big"
 	"math/rand"
 	"os"
 	"time"
 
+	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/testground/sdk-go/sync"
+
 	"github.com/filecoin-project/go-address"
+	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/lotus/api"
-	"github.com/testground/sdk-go/sync"
-
-	mbig "math/big"
-
 	"github.com/filecoin-project/lotus/build"
-
 	"github.com/filecoin-project/lotus/testplans/lotus-soup/testkit"
 )
 
@@ -39,6 +39,8 @@ import (
 // Then we create a genesis block that allocates some funds to each node and collects
 // the presealed sectors.
 func dealsE2E(t *testkit.TestEnvironment) error {
+	t.RecordMessage("running node with role '%s'", t.Role)
+
 	// Dispatch/forward non-client roles to defaults.
 	if t.Role != "client" {
 		return testkit.HandleDefaultRole(t)
@@ -79,7 +81,7 @@ func dealsE2E(t *testkit.TestEnvironment) error {
 
 	time.Sleep(time.Duration(t.GlobalSeq) * 5 * time.Second)
 
-	// generate 1600 bytes of random data
+	// generate 5000000 bytes of random data
 	data := make([]byte, 5000000)
 	rand.New(rand.NewSource(time.Now().UnixNano())).Read(data)
 
@@ -100,6 +102,15 @@ func dealsE2E(t *testkit.TestEnvironment) error {
 	}
 	t.RecordMessage("file cid: %s", fcid)
 
+	// Check if we should bounce the connection during data transfers
+	if t.BooleanParam("bounce_conn_data_transfers") {
+		t.RecordMessage("Will bounce connection during push and pull data-transfers")
+		err = bounceConnInTransfers(ctx, t, client, minerAddr.MinerNetAddrs.ID)
+		if err != nil {
+			return err
+		}
+	}
+
 	// start deal
 	t1 := time.Now()
 	deal := testkit.StartDeal(ctx, minerAddr.MinerActorAddr, client, fcid.Root, fastRetrieval)
@@ -133,6 +144,55 @@ func dealsE2E(t *testkit.TestEnvironment) error {
 	return nil
 }
 
+func bounceConnInTransfers(ctx context.Context, t *testkit.TestEnvironment, client api.FullNode, minerPeerID peer.ID) error {
+	storageConnBroken := false
+	retrievalConnBroken := false
+	upds, err := client.ClientDataTransferUpdates(ctx)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		for upd := range upds {
+			dir := "push"
+			if !upd.IsSender {
+				dir = "pull"
+			}
+
+			t.RecordMessage("%s data transfer status: %s, transferred: %d", dir, datatransfer.Statuses[upd.Status], upd.Transferred)
+
+			// Bounce the connection after the first block is sent for the storage deal
+			if upd.IsSender && upd.Transferred > 0 && !storageConnBroken {
+				storageConnBroken = true
+				bounceConnection(ctx, t, client, minerPeerID)
+			}
+
+			// Bounce the connection after the first block is received for the retrieval deal
+			if !upd.IsSender && upd.Transferred > 0 && !retrievalConnBroken {
+				retrievalConnBroken = true
+				bounceConnection(ctx, t, client, minerPeerID)
+			}
+		}
+	}()
+
+	return nil
+}
+
+func bounceConnection(ctx context.Context, t *testkit.TestEnvironment, client api.FullNode, minerPeerID peer.ID) {
+	t.RecordMessage("disconnecting peer %s", minerPeerID)
+	client.NetBlockAdd(ctx, api.NetBlockList{
+		Peers: []peer.ID{minerPeerID},
+	})
+
+	go func() {
+		time.Sleep(3 * time.Second)
+		t.RecordMessage("reconnecting to peer %s", minerPeerID)
+		client.NetBlockRemove(ctx, api.NetBlockList{
+			Peers: []peer.ID{minerPeerID},
+		})
+	}()
+}
+
 // filToAttoFil converts a fractional filecoin value into AttoFIL, rounding if necessary
 func filToAttoFil(f float64) big.Int {
 	a := mbig.NewFloat(f)

testplans/lotus-soup/go.mod

@@ -8,11 +8,12 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/drand/drand v1.2.1
 	github.com/filecoin-project/go-address v0.0.5
-	github.com/filecoin-project/go-fil-markets v1.2.5
+	github.com/filecoin-project/go-data-transfer v1.1.1-0.20210428151930-29bfef7e037e
+	github.com/filecoin-project/go-fil-markets v1.3.0-rc1.0.20210428152617-25f4f7791e17
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-state-types v0.1.0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/lotus v1.8.1-0.20210428122447-4688da51781a
+	github.com/filecoin-project/lotus v1.6.1-0.20210429092235-20782ecae36f
 	github.com/filecoin-project/specs-actors v0.9.13
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.4

testplans/lotus-soup/go.sum

@@ -101,6 +101,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
+github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@@ -273,16 +275,16 @@ github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZ
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo=
-github.com/filecoin-project/go-data-transfer v1.4.3 h1:ECEw69NOfmEZ7XN1NSBvj3KTbbH2mIczQs+Z2w4bD7c=
-github.com/filecoin-project/go-data-transfer v1.4.3/go.mod h1:n8kbDQXWrY1c4UgfMa9KERxNCWbOTDwdNhf2MpN9dpo=
+github.com/filecoin-project/go-data-transfer v1.1.1-0.20210428151930-29bfef7e037e h1:8sGyac9gEAPRUifBYQfEdNqPUS6S0p4Eh+D1ATDgmIw=
+github.com/filecoin-project/go-data-transfer v1.1.1-0.20210428151930-29bfef7e037e/go.mod h1:E3WW4mCEYwU2y65swPEajSZoFWFmfXt7uwGduoACZQc=
 github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ=
 github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.2.5 h1:bQgtXbwxKyPxSEQoUI5EaTHJ0qfzyd5NosspuADCm6Y=
-github.com/filecoin-project/go-fil-markets v1.2.5/go.mod h1:7JIqNBmFvOyBzk/EiPYnweVdQnWhshixb5B9b1653Ag=
+github.com/filecoin-project/go-fil-markets v1.3.0-rc1.0.20210428152617-25f4f7791e17 h1:32eSVd/b6+Y0I+bx3OHAE5x3QggdK7Te4Ysv5rFUtSI=
+github.com/filecoin-project/go-fil-markets v1.3.0-rc1.0.20210428152617-25f4f7791e17/go.mod h1:bYo+LdtoDRs1KLtogTHty1ioFFJDjf6mEVmYPT6dW/A=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -310,8 +312,8 @@ github.com/filecoin-project/go-statestore v0.1.1 h1:ufMFq00VqnT2CAuDpcGnwLnCX1I/
 github.com/filecoin-project/go-statestore v0.1.1/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
-github.com/filecoin-project/lotus v1.8.1-0.20210428122447-4688da51781a h1:8M4BZHB+R3psW72CGarBdyc5OvbFBRRlebzw8u5EWsg=
-github.com/filecoin-project/lotus v1.8.1-0.20210428122447-4688da51781a/go.mod h1:4YC/8rizrrp2wKOYvHQEjCxZbziXi68BhrzvI+FCye0=
+github.com/filecoin-project/lotus v1.6.1-0.20210429092235-20782ecae36f h1:e85TP82iW8oMUCt/hdbpAExiIQrez8LHJay0bDi4ITQ=
+github.com/filecoin-project/lotus v1.6.1-0.20210429092235-20782ecae36f/go.mod h1:Sp9hSZZXileBDMt8rQMYTy2/c+0cywaYvetKtuWALk0=
 github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4=
 github.com/filecoin-project/specs-actors v0.9.12/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
 github.com/filecoin-project/specs-actors v0.9.13 h1:rUEOQouefi9fuVY/2HOroROJlZbOzWYXXeIh41KF2M4=
@@ -322,8 +324,6 @@ github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb
 github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb/go.mod h1:LljnY2Mn2homxZsmokJZCpRuhOPxfXhvcek5gWkmqAc=
 github.com/filecoin-project/specs-actors/v3 v3.1.0 h1:s4qiPw8pgypqBGAy853u/zdZJ7K9cTZdM1rTiSonHrg=
 github.com/filecoin-project/specs-actors/v3 v3.1.0/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww=
-github.com/filecoin-project/specs-actors/v4 v4.0.0 h1:vMALksY5G3J5rj3q9rbcyB+f4Tk1xrLqSgdB3jOok4s=
-github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng=
 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506 h1:Ur/l2+6qN+lQiqjozWWc5p9UDaAMDZKTlDS98oRnlIw=
 github.com/filecoin-project/specs-storage v0.1.1-0.20201105051918-5188d9774506/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
 github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
@@ -620,8 +620,9 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28
 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
 github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0=
 github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY=
-github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0=
-github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
+github.com/ipfs/go-graphsync v0.6.1/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
+github.com/ipfs/go-graphsync v0.6.2-0.20210428121800-88edb5462e17 h1:rOoF88dVuDGbIx7idSdimN7JvXriyOIT96WD3eX9sHA=
+github.com/ipfs/go-graphsync v0.6.2-0.20210428121800-88edb5462e17/go.mod h1:5WyaeigpNdpiYQuW2vwpuecOoEfB4h747ZGEOKmAGTg=
 github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=

testplans/lotus-soup/manifest.toml

@@ -58,6 +58,9 @@ instances = { min = 1, max = 100, default = 5 }
   # Fast retrieval
   fast_retrieval = { type = "bool", default = false }
 
+  # Bounce connection during push and pull data transfers
+  bounce_conn_data_transfers = { type = "bool", default = false }
+
 
 [[testcases]]
 name = "drand-halting"

testplans/lotus-soup/testkit/net.go

@@ -74,6 +74,11 @@ func ApplyNetworkParameters(t *TestEnvironment) {
 		t.D().RecordPoint("duplicate_packet_correlation", float64(ls.DuplicateCorr))
 	}
 
+	if t.IsParamSet("bandwidth") {
+		ls.Bandwidth = t.SizeParam("bandwidth")
+		t.D().RecordPoint("bandwidth_bytes", float64(ls.Bandwidth))
+	}
+
 	t.NetClient.MustConfigureNetwork(ctx, &network.Config{
 		Network:        "default",
 		Enable:         true,