From 5427eb7a0cc9558e4c61eeeaab9cc58408fa8931 Mon Sep 17 00:00:00 2001 From: Alex Cruikshank <169613+acruikshank@users.noreply.github.com> Date: Wed, 14 Aug 2019 16:54:31 -0500 Subject: [PATCH] add seal-now command (#3231) * add seal-now command * revert message send argument fix * fix import * minor improvements and corrections * simplify test * make seal now test a functional test * run miner async --- commands/mining.go | 24 ++++-- commands/mining_daemon_test.go | 57 ++++++++++++++ porcelain/api.go | 5 ++ porcelain/sectorbuilder.go | 36 ++++++++- porcelain/sectorbuilder_test.go | 94 +++++++++++++++++++++++ proofs/sectorbuilder/interface.go | 4 + proofs/sectorbuilder/rustsectorbuilder.go | 6 +- tools/fast/action_mining.go | 14 ++++ 8 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 porcelain/sectorbuilder_test.go diff --git a/commands/mining.go b/commands/mining.go index df41cafd2c..3a69143749 100644 --- a/commands/mining.go +++ b/commands/mining.go @@ -19,11 +19,12 @@ var miningCmd = &cmds.Command{ Tagline: "Manage all mining operations for a node", }, Subcommands: map[string]*cmds.Command{ - "address": miningAddrCmd, - "once": miningOnceCmd, - "start": miningStartCmd, - "status": miningStatusCmd, - "stop": miningStopCmd, + "address": miningAddrCmd, + "once": miningOnceCmd, + "start": miningStartCmd, + "status": miningStatusCmd, + "stop": miningStopCmd, + "seal-now": miningSealCmd, }, } @@ -162,6 +163,19 @@ var miningStopCmd = &cmds.Command{ Encoders: stringEncoderMap, } +var miningSealCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Start sealing all staged sectors or create and seal a new sector", + }, + Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) error { + if err := GetPorcelainAPI(env).SealNow(req.Context); err != nil { + return err + } + return re.Emit("sealing started") + }, + Encoders: stringEncoderMap, +} + var stringEncoderMap = cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, t string) error { fmt.Fprintln(w, t) // nolint: errcheck diff --git a/commands/mining_daemon_test.go b/commands/mining_daemon_test.go index 4891e9f280..9c79270a60 100644 --- a/commands/mining_daemon_test.go +++ b/commands/mining_daemon_test.go @@ -1,14 +1,21 @@ package commands_test import ( + "context" "math/big" "strings" "testing" + "time" + "github.com/filecoin-project/go-filecoin/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/filecoin-project/go-filecoin/fixtures" tf "github.com/filecoin-project/go-filecoin/testhelpers/testflags" + "github.com/filecoin-project/go-filecoin/tools/fast" + "github.com/filecoin-project/go-filecoin/tools/fast/fastesting" + "github.com/filecoin-project/go-filecoin/tools/fast/series" ) func parseInt(t *testing.T, s string) *big.Int { @@ -37,3 +44,53 @@ func TestMiningGenBlock(t *testing.T) { assert.Equal(t, sum.Add(beforeBalance, big.NewInt(1000)), afterBalance) } + +func TestMiningSealNow(t *testing.T) { + tf.FunctionalTest(t) + + ctx, env := fastesting.NewTestEnvironment(context.Background(), t, fast.FilecoinOpts{ + InitOpts: []fast.ProcessInitOption{fast.POAutoSealIntervalSeconds(1)}, + DaemonOpts: []fast.ProcessDaemonOption{fast.POBlockTime(50 * time.Millisecond)}, + }) + env.RunAsyncMiner() + defer func() { + require.NoError(t, env.Teardown(ctx)) + }() + + genesisNode := env.GenesisMiner + require.NoError(t, genesisNode.MiningStart(ctx)) + defer func() { + require.NoError(t, genesisNode.MiningStop(ctx)) + }() + + minerNode := env.RequireNewNodeWithFunds(1000) + + // Connect the clientNode and the minerNode + require.NoError(t, series.Connect(ctx, genesisNode, minerNode)) + + // Calls MiningOnce on genesis (client). This also starts the Miner. + _, err := series.CreateStorageMinerWithAsk(ctx, minerNode, big.NewInt(500), big.NewFloat(0.0001), big.NewInt(3000)) + require.NoError(t, err) + + // get address of miner so we can check power + miningAddress, err := minerNode.MiningAddress(ctx) + require.NoError(t, err) + + // start sealing + err = minerNode.SealNow(ctx) + require.NoError(t, err) + + // We know the miner has sealed and committed a sector if their power increases on chain. + // Wait up to 3 minutes for that to happen. + for i := 0; i < 180; i++ { + power, err := minerNode.MinerPower(ctx, miningAddress) + require.NoError(t, err) + + if power.Power.GreaterThan(types.ZeroBytes) { + // miner has gained power, so seal was successful + return + } + time.Sleep(time.Second) + } + assert.Fail(t, "timed out waiting for miner to gain power from sealing") +} diff --git a/porcelain/api.go b/porcelain/api.go index 4583f9b699..8c61e11994 100644 --- a/porcelain/api.go +++ b/porcelain/api.go @@ -240,6 +240,11 @@ func (a *API) CalculatePoSt(ctx context.Context, sortedCommRs proofs.SortedCommR return CalculatePoSt(ctx, a, sortedCommRs, seed) } +// SealNow forces the sectorbuilder to either seal the staged sectors it has or create a new one and seal it immediately +func (a *API) SealNow(ctx context.Context) error { + return SealNow(ctx, a) +} + // PingMinerWithTimeout pings a storage or retrieval miner, waiting the given // timeout and returning desciptive errors. func (a *API) PingMinerWithTimeout( diff --git a/porcelain/sectorbuilder.go b/porcelain/sectorbuilder.go index 4069f4dc38..cccc50888d 100644 --- a/porcelain/sectorbuilder.go +++ b/porcelain/sectorbuilder.go @@ -1,12 +1,16 @@ package porcelain import ( + "bytes" "context" - "errors" "github.com/filecoin-project/go-filecoin/proofs" "github.com/filecoin-project/go-filecoin/proofs/sectorbuilder" "github.com/filecoin-project/go-filecoin/types" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + + "github.com/pkg/errors" ) type sbPlumbing interface { @@ -29,3 +33,33 @@ func CalculatePoSt(ctx context.Context, plumbing sbPlumbing, sortedCommRs proofs } return res.Proofs, res.Faults, nil } + +// SealNow forces the sectorbuilder to either seal the staged sectors it has or create a new one and seal it immediately +func SealNow(ctx context.Context, plumbing sbPlumbing) error { + if plumbing.SectorBuilder() == nil { + return errors.New("must be mining to seal sectors") + } + + stagedSectors, err := plumbing.SectorBuilder().GetAllStagedSectors() + if err != nil { + return errors.Wrap(err, "could not retrieved staged sectors") + } + + // if no sectors are staged, add a 1 byte piece to ensure at least one seal + if len(stagedSectors) == 0 { + data := []byte{0} + hash, err := multihash.Sum(data, multihash.SHA2_256, -1) + if err != nil { + return errors.Wrap(err, "could not create cid for piece") + } + pieceRef := cid.NewCidV1(cid.DagCBOR, hash) + _, err = plumbing.SectorBuilder().AddPiece(ctx, pieceRef, 1, bytes.NewReader(data)) + if err != nil { + return errors.Wrap(err, "could not add piece to trigger sealing") + } + + } + + // start sealing on all existing staged sectors + return plumbing.SectorBuilder().SealAllStagedSectors(ctx) +} diff --git a/porcelain/sectorbuilder_test.go b/porcelain/sectorbuilder_test.go new file mode 100644 index 0000000000..bf0735bcd3 --- /dev/null +++ b/porcelain/sectorbuilder_test.go @@ -0,0 +1,94 @@ +package porcelain_test + +import ( + "context" + "io" + "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + . "github.com/filecoin-project/go-filecoin/porcelain" + "github.com/filecoin-project/go-filecoin/proofs/sectorbuilder" + "github.com/filecoin-project/go-sectorbuilder" +) + +func TestSealNow(t *testing.T) { + t.Run("adds piece and triggers sealing when staged sectors is empty", func(t *testing.T) { + p := newTestSectorBuilderPlumbing(0) + + err := SealNow(context.Background(), p) + require.NoError(t, err) + + // adds a piece + assert.Equal(t, 1, p.sectorbuilder.addPieceCount) + + // seals sectors + assert.Equal(t, 1, p.sectorbuilder.sealAllSectorsCount) + }) + + t.Run("does not add a piece when staged sectors exist", func(t *testing.T) { + p := newTestSectorBuilderPlumbing(4) + + err := SealNow(context.Background(), p) + require.NoError(t, err) + + // does not add a piece + assert.Equal(t, 0, p.sectorbuilder.addPieceCount) + + // seals sectors + assert.Equal(t, 1, p.sectorbuilder.sealAllSectorsCount) + }) +} + +func newTestSectorBuilderPlumbing(stagedSectors int) *testSectorBuilderPlumbing { + sb := &testSectorBuilder{numStagedSectors: stagedSectors} + return &testSectorBuilderPlumbing{ + sectorbuilder: sb, + } +} + +type testSectorBuilderPlumbing struct { + sectorbuilder *testSectorBuilder +} + +func (tsbp *testSectorBuilderPlumbing) SectorBuilder() sectorbuilder.SectorBuilder { + return tsbp.sectorbuilder +} + +type testSectorBuilder struct { + addPieceCount int + sealAllSectorsCount int + numStagedSectors int +} + +func (tsb *testSectorBuilder) AddPiece(ctx context.Context, pieceRef cid.Cid, pieceSize uint64, pieceReader io.Reader) (sectorID uint64, err error) { + tsb.addPieceCount++ + return 0, nil +} + +func (tsb *testSectorBuilder) ReadPieceFromSealedSector(pieceCid cid.Cid) (io.Reader, error) { + return nil, nil +} + +func (tsb *testSectorBuilder) SealAllStagedSectors(ctx context.Context) error { + tsb.sealAllSectorsCount++ + return nil +} + +func (tsb *testSectorBuilder) GetAllStagedSectors() ([]go_sectorbuilder.StagedSectorMetadata, error) { + return make([]go_sectorbuilder.StagedSectorMetadata, tsb.numStagedSectors), nil +} + +func (tsb *testSectorBuilder) SectorSealResults() <-chan sectorbuilder.SectorSealResult { + return nil +} + +func (tsb *testSectorBuilder) GeneratePoSt(sectorbuilder.GeneratePoStRequest) (sectorbuilder.GeneratePoStResponse, error) { + return sectorbuilder.GeneratePoStResponse{}, nil +} + +func (tsb *testSectorBuilder) Close() error { + return nil +} diff --git a/proofs/sectorbuilder/interface.go b/proofs/sectorbuilder/interface.go index 5ae80b74dc..8fb9fa8c6e 100644 --- a/proofs/sectorbuilder/interface.go +++ b/proofs/sectorbuilder/interface.go @@ -4,6 +4,7 @@ import ( "context" "io" + "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" @@ -33,6 +34,9 @@ type SectorBuilder interface { // SealAllStagedSectors seals any non-empty staged sectors. SealAllStagedSectors(ctx context.Context) error + // GetAllStagedSectors returns a slice of all staged sector metadata for the sector builder, or an error. + GetAllStagedSectors() ([]go_sectorbuilder.StagedSectorMetadata, error) + // SectorSealResults returns an unbuffered channel that is sent a value // whenever sealing completes. All calls to SectorSealResults will get the // same channel. Values will be either a *SealedSectorMetadata or an error. diff --git a/proofs/sectorbuilder/rustsectorbuilder.go b/proofs/sectorbuilder/rustsectorbuilder.go index 5ddea34834..acf8065335 100644 --- a/proofs/sectorbuilder/rustsectorbuilder.go +++ b/proofs/sectorbuilder/rustsectorbuilder.go @@ -70,7 +70,7 @@ func NewRustSectorBuilder(cfg RustSectorBuilderConfig) (*RustSectorBuilder, erro } // load staged sector metadata and use it to initialize the poller - metadata, err := sb.stagedSectors() + metadata, err := sb.GetAllStagedSectors() if err != nil { return nil, errors.Wrap(err, "failed to load staged sectors") } @@ -232,8 +232,8 @@ func (sb *RustSectorBuilder) SealAllStagedSectors(ctx context.Context) error { return go_sectorbuilder.SealAllStagedSectors(sb.ptr) } -// stagedSectors returns a slice of all staged sector metadata for the sector builder, or an error. -func (sb *RustSectorBuilder) stagedSectors() ([]go_sectorbuilder.StagedSectorMetadata, error) { +// GetAllStagedSectors returns a slice of all staged sector metadata for the sector builder, or an error. +func (sb *RustSectorBuilder) GetAllStagedSectors() ([]go_sectorbuilder.StagedSectorMetadata, error) { return go_sectorbuilder.GetAllStagedSectors(sb.ptr) } diff --git a/tools/fast/action_mining.go b/tools/fast/action_mining.go index 34197566ef..b167acd054 100644 --- a/tools/fast/action_mining.go +++ b/tools/fast/action_mining.go @@ -69,3 +69,17 @@ func (f *Filecoin) MiningStatus(ctx context.Context) (commands.MiningStatusResul return out, nil } + +// SealNow adds a staged sector if none exists and then triggers sealing on it +func (f *Filecoin) SealNow(ctx context.Context) error { + out, err := f.RunCmdWithStdin(ctx, nil, "go-filecoin", "mining", "seal-now") + if err != nil { + return err + } + + if out.ExitCode() > 0 { + return fmt.Errorf("filecoin command: %s, exited with non-zero exitcode: %d", out.Args(), out.ExitCode()) + } + + return nil +}